fixes #21146

extensions/git/src/encoding.ts

@@ -0,0 +1,81 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as jschardet from 'jschardet';
+
+jschardet.Constants.MINIMUM_THRESHOLD = 0.2;
+
+function detectEncodingByBOM(buffer: NodeBuffer): string | null {
+	if (!buffer || buffer.length < 2) {
+		return null;
+	}
+
+	const b0 = buffer.readUInt8(0);
+	const b1 = buffer.readUInt8(1);
+
+	// UTF-16 BE
+	if (b0 === 0xFE && b1 === 0xFF) {
+		return 'utf16be';
+	}
+
+	// UTF-16 LE
+	if (b0 === 0xFF && b1 === 0xFE) {
+		return 'utf16le';
+	}
+
+	if (buffer.length < 3) {
+		return null;
+	}
+
+	const b2 = buffer.readUInt8(2);
+
+	// UTF-8
+	if (b0 === 0xEF && b1 === 0xBB && b2 === 0xBF) {
+		return 'utf8';
+	}
+
+	return null;
+}
+
+const IGNORE_ENCODINGS = [
+	'ascii',
+	'utf-8',
+	'utf-16',
+	'utf-32'
+];
+
+const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = {
+	'ibm866': 'cp866',
+	'big5': 'cp950'
+};
+
+export function detectEncoding(buffer: Buffer): string | null {
+	let result = detectEncodingByBOM(buffer);
+
+	if (result) {
+		return result;
+	}
+
+	const detected = jschardet.detect(buffer);
+
+	if (!detected || !detected.encoding) {
+		return null;
+	}
+
+	const encoding = detected.encoding;
+
+	// Ignore encodings that cannot guess correctly
+	// (http://chardet.readthedocs.io/en/latest/supported-encodings.html)
+	if (0 <= IGNORE_ENCODINGS.indexOf(encoding.toLowerCase())) {
+		return null;
+	}
+
+	const normalizedEncodingName = encoding.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+	const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName];
+
+	return mapped || normalizedEncodingName;
+}

extensions/git/src/git.ts

@@ -15,6 +15,7 @@ import iconv = require('iconv-lite');
 import * as filetype from 'file-type';
 import { assign, uniqBy, groupBy, denodeify, IDisposable, toDisposable, dispose, mkdirp, readBytes, detectUnicodeEncoding, Encoding, onceEvent } from './util';
 import { CancellationToken } from 'vscode';
+import { detectEncoding } from './encoding';
 
 const readfile = denodeify<string, string | null, string>(fs.readFile);
 
@@ -661,7 +662,10 @@ export class Repository {
 
 	async bufferString(object: string, encoding: string = 'utf8'): Promise<string> {
 		const stdout = await this.buffer(object);
-		return iconv.decode(stdout, iconv.encodingExists(encoding) ? encoding : 'utf8');
+		encoding = detectEncoding(stdout) || encoding;
+		encoding = iconv.encodingExists(encoding) ? encoding : 'utf8';
+
+		return iconv.decode(stdout, encoding);
 	}
 
 	async buffer(object: string): Promise<Buffer> {

extensions/git/src/repository.ts

@@ -811,24 +811,18 @@ export class Repository implements Disposable {
 	}
 
 	async show(ref: string, filePath: string): Promise<string> {
-		return await this.run(Operation.Show, async () => {
+		return this.run(Operation.Show, () => {
 			const relativePath = path.relative(this.repository.root, filePath).replace(/\\/g, '/');
 			const configFiles = workspace.getConfiguration('files', Uri.file(filePath));
-			const encoding = configFiles.get<string>('encoding');
-
-			// TODO@joao: Resource config api
-			return await this.repository.bufferString(`${ref}:${relativePath}`, encoding);
+			const defaultEncoding = configFiles.get<string>('encoding');
+			return this.repository.bufferString(`${ref}:${relativePath}`, defaultEncoding);
 		});
 	}
 
 	async buffer(ref: string, filePath: string): Promise<Buffer> {
-		return await this.run(Operation.Show, async () => {
+		return this.run(Operation.Show, () => {
 			const relativePath = path.relative(this.repository.root, filePath).replace(/\\/g, '/');
-			// const configFiles = workspace.getConfiguration('files', Uri.file(filePath));
-			// const encoding = configFiles.get<string>('encoding');
-
-			// TODO@joao: REsource config api
-			return await this.repository.buffer(`${ref}:${relativePath}`);
+			return this.repository.buffer(`${ref}:${relativePath}`);
 		});
 	}
 

extensions/git/src/typings/jschardet.d.ts

@@ -0,0 +1,11 @@
+declare module 'jschardet' {
+	export interface IDetectedMap {
+		encoding: string,
+		confidence: number
+	}
+	export function detect(buffer: NodeBuffer): IDetectedMap;
+
+	export const Constants: {
+		MINIMUM_THRESHOLD: number,
+	}
+}