Use TextDecoder when decoding UTF-8 (#102369)

* textfiles - always use TextDecoder when reading UTF-8

* fix tests

* do not share a text decoder when using streams

* jsdoc

src/vs/workbench/services/textfile/common/encoding.ts

@@ -49,18 +49,39 @@ class DecoderStream implements IDecoderStream {
 	 * This stream will only load iconv-lite lazily if the encoding
 	 * is not UTF-8. This ensures that for most common cases we do
 	 * not pay the price of loading the module from disk.
+	 *
+	 * We still need to be careful when converting UTF-8 to a string
+	 * though because we read the file in chunks of Buffer and thus
+	 * need to decode it via TextDecoder helper that is available
+	 * in browser and node.js environments.
 	 */
 	static async create(encoding: string): Promise<DecoderStream> {
 		let decoder: IDecoderStream | undefined = undefined;
 		if (encoding !== UTF8) {
 			const iconv = await import('iconv-lite-umd');
 			decoder = iconv.getDecoder(toNodeEncoding(encoding));
+		} else {
+			const utf8TextDecoder = new TextDecoder();
+			decoder = {
+				write(buffer: Uint8Array): string {
+					return utf8TextDecoder.decode(buffer, {
+						// Signal to TextDecoder that potentially more data is coming
+						// and that we are calling `decode` in the end to consume any
+						// remainders
+						stream: true
+					});
+				},
+
+				end(): string | undefined {
+					return utf8TextDecoder.decode();
+				}
+			};
 		}
 
 		return new DecoderStream(decoder);
 	}
 
-	private constructor(private iconvLiteDecoder: IDecoderStream | undefined) { }
+	private constructor(private iconvLiteDecoder: IDecoderStream) { }
 
 	write(buffer: Uint8Array): string {
 		if (this.iconvLiteDecoder) {

src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts

@@ -335,7 +335,7 @@ suite('Encoding', () => {
 		const source = newTestReadableStream(buffers);
 		const { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
 
-		const expected = incompleteEmojis.toString(encoding.UTF8);
+		const expected = new TextDecoder().decode(incompleteEmojis);
 		const actual = await readAllAsString(stream);
 
 		assert.equal(actual, expected);