diff --git a/src/vs/workbench/services/textfile/common/encoding.ts b/src/vs/workbench/services/textfile/common/encoding.ts index fa1fa41f336..0a6428ae998 100644 --- a/src/vs/workbench/services/textfile/common/encoding.ts +++ b/src/vs/workbench/services/textfile/common/encoding.ts @@ -49,18 +49,39 @@ class DecoderStream implements IDecoderStream { * This stream will only load iconv-lite lazily if the encoding * is not UTF-8. This ensures that for most common cases we do * not pay the price of loading the module from disk. + * + * We still need to be careful when converting UTF-8 to a string + * though because we read the file in chunks of Buffer and thus + * need to decode it via TextDecoder helper that is available + * in browser and node.js environments. */ static async create(encoding: string): Promise { let decoder: IDecoderStream | undefined = undefined; if (encoding !== UTF8) { const iconv = await import('iconv-lite-umd'); decoder = iconv.getDecoder(toNodeEncoding(encoding)); + } else { + const utf8TextDecoder = new TextDecoder(); + decoder = { + write(buffer: Uint8Array): string { + return utf8TextDecoder.decode(buffer, { + // Signal to TextDecoder that potentially more data is coming + // and that we are calling `decode` in the end to consume any + // remainders + stream: true + }); + }, + + end(): string | undefined { + return utf8TextDecoder.decode(); + } + }; } return new DecoderStream(decoder); } - private constructor(private iconvLiteDecoder: IDecoderStream | undefined) { } + private constructor(private iconvLiteDecoder: IDecoderStream) { } write(buffer: Uint8Array): string { if (this.iconvLiteDecoder) { diff --git a/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts b/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts index bc4af904ca4..e113f8b4f2b 100644 --- a/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts +++ b/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts @@ -335,7 +335,7 @@ suite('Encoding', () => { const source = newTestReadableStream(buffers); const { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); - const expected = incompleteEmojis.toString(encoding.UTF8); + const expected = new TextDecoder().decode(incompleteEmojis); const actual = await readAllAsString(stream); assert.equal(actual, expected);