diff --git a/extensions/git/package-lock.json b/extensions/git/package-lock.json index bc150555c70..7a9cb2aded8 100644 --- a/extensions/git/package-lock.json +++ b/extensions/git/package-lock.json @@ -14,7 +14,6 @@ "@vscode/iconv-lite-umd": "0.7.0", "byline": "^5.0.0", "file-type": "16.5.4", - "jschardet": "3.1.4", "picomatch": "2.3.1", "vscode-uri": "^2.0.0", "which": "4.0.0" @@ -279,15 +278,6 @@ "node": ">=16" } }, - "node_modules/jschardet": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/jschardet/-/jschardet-3.1.4.tgz", - "integrity": "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg==", - "license": "LGPL-2.1+", - "engines": { - "node": ">=0.1.90" - } - }, "node_modules/peek-readable": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", diff --git a/extensions/git/package.json b/extensions/git/package.json index f5195eee8fb..06f40d948a1 100644 --- a/extensions/git/package.json +++ b/extensions/git/package.json @@ -35,6 +35,7 @@ "statusBarItemTooltip", "tabInputMultiDiff", "tabInputTextMerge", + "textDocumentEncoding", "textEditorDiffInformation", "timeline" ], @@ -3566,7 +3567,6 @@ "@vscode/iconv-lite-umd": "0.7.0", "byline": "^5.0.0", "file-type": "16.5.4", - "jschardet": "3.1.4", "picomatch": "2.3.1", "vscode-uri": "^2.0.0", "which": "4.0.0" diff --git a/extensions/git/src/commands.ts b/extensions/git/src/commands.ts index 94bcb143c58..b09e6d5a6c5 100644 --- a/extensions/git/src/commands.ts +++ b/extensions/git/src/commands.ts @@ -1628,19 +1628,29 @@ export class CommandCenter { } let modifiedUri = changes.modifiedUri; + let modifiedDocument: TextDocument | undefined; + if (!modifiedUri) { const textEditor = window.activeTextEditor; if (!textEditor) { return; } - const modifiedDocument = textEditor.document; + + modifiedDocument = textEditor.document; modifiedUri = modifiedDocument.uri; } + if (modifiedUri.scheme !== 'file') { return; } + + if (!modifiedDocument) { + modifiedDocument = await workspace.openTextDocument(modifiedUri); + } + const result = changes.originalWithModifiedChanges; - await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result)); + await this.runByRepository(modifiedUri, async (repository, resource) => + await repository.stage(resource, result, modifiedDocument.encoding)); } @command('git.stageSelectedRanges', { diff: true }) @@ -1817,7 +1827,8 @@ export class CommandCenter { const originalDocument = await workspace.openTextDocument(originalUri); const result = applyLineChanges(originalDocument, modifiedDocument, changes); - await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result)); + await this.runByRepository(modifiedUri, async (repository, resource) => + await repository.stage(resource, result, modifiedDocument.encoding)); } @command('git.revertChange') @@ -1989,7 +2000,7 @@ export class CommandCenter { this.logger.trace(`[CommandCenter][unstageSelectedRanges] invertedDiffs: ${JSON.stringify(invertedDiffs)}`); const result = applyLineChanges(modifiedDocument, originalDocument, invertedDiffs); - await repository.stage(modifiedUri, result); + await repository.stage(modifiedUri, result, modifiedDocument.encoding); } @command('git.unstageFile') diff --git a/extensions/git/src/encoding.ts b/extensions/git/src/encoding.ts deleted file mode 100644 index c80fb6ee6d5..00000000000 --- a/extensions/git/src/encoding.ts +++ /dev/null @@ -1,100 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as jschardet from 'jschardet'; - -function detectEncodingByBOM(buffer: Buffer): string | null { - if (!buffer || buffer.length < 2) { - return null; - } - - const b0 = buffer.readUInt8(0); - const b1 = buffer.readUInt8(1); - - // UTF-16 BE - if (b0 === 0xFE && b1 === 0xFF) { - return 'utf16be'; - } - - // UTF-16 LE - if (b0 === 0xFF && b1 === 0xFE) { - return 'utf16le'; - } - - if (buffer.length < 3) { - return null; - } - - const b2 = buffer.readUInt8(2); - - // UTF-8 - if (b0 === 0xEF && b1 === 0xBB && b2 === 0xBF) { - return 'utf8'; - } - - return null; -} - -const IGNORE_ENCODINGS = [ - 'ascii', - 'utf-8', - 'utf-16', - 'utf-32' -]; - -const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = { - 'ibm866': 'cp866', - 'big5': 'cp950' -}; - -const MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET: { [key: string]: string } = { - utf8: 'UTF-8', - utf16le: 'UTF-16LE', - utf16be: 'UTF-16BE', - windows1252: 'windows-1252', - windows1250: 'windows-1250', - iso88592: 'ISO-8859-2', - windows1251: 'windows-1251', - cp866: 'IBM866', - iso88595: 'ISO-8859-5', - koi8r: 'KOI8-R', - windows1253: 'windows-1253', - iso88597: 'ISO-8859-7', - windows1255: 'windows-1255', - iso88598: 'ISO-8859-8', - cp950: 'Big5', - shiftjis: 'SHIFT_JIS', - eucjp: 'EUC-JP', - euckr: 'EUC-KR', - gb2312: 'GB2312' -}; - -export function detectEncoding(buffer: Buffer, candidateGuessEncodings: string[]): string | null { - const result = detectEncodingByBOM(buffer); - - if (result) { - return result; - } - - candidateGuessEncodings = candidateGuessEncodings.map(e => MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET[e]).filter(e => !!e); - - const detected = jschardet.detect(buffer, candidateGuessEncodings.length > 0 ? { detectEncodings: candidateGuessEncodings } : undefined); - if (!detected || !detected.encoding) { - return null; - } - - const encoding = detected.encoding; - - // Ignore encodings that cannot guess correctly - // (http://chardet.readthedocs.io/en/latest/supported-encodings.html) - if (0 <= IGNORE_ENCODINGS.indexOf(encoding.toLowerCase())) { - return null; - } - - const normalizedEncodingName = encoding.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(); - const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName]; - - return mapped || normalizedEncodingName; -} diff --git a/extensions/git/src/git.ts b/extensions/git/src/git.ts index 2f0d11bd488..29fc38abfed 100644 --- a/extensions/git/src/git.ts +++ b/extensions/git/src/git.ts @@ -14,7 +14,6 @@ import * as iconv from '@vscode/iconv-lite-umd'; import * as filetype from 'file-type'; import { assign, groupBy, IDisposable, toDisposable, dispose, mkdirp, readBytes, detectUnicodeEncoding, Encoding, onceEvent, splitInChunks, Limiter, Versions, isWindows, pathEquals, isMacintosh, isDescendant } from './util'; import { CancellationError, CancellationToken, ConfigurationChangeEvent, LogOutputChannel, Progress, Uri, workspace } from 'vscode'; -import { detectEncoding } from './encoding'; import { Ref, RefType, Branch, Remote, ForcePushMode, GitErrorCodes, LogOptions, Change, Status, CommitOptions, RefQuery, InitOptions } from './api/git'; import * as byline from 'byline'; import { StringDecoder } from 'string_decoder'; @@ -1330,18 +1329,6 @@ export class Repository { .filter(entry => !!entry); } - async bufferString(object: string, encoding: string = 'utf8', autoGuessEncoding = false, candidateGuessEncodings: string[] = []): Promise { - const stdout = await this.buffer(object); - - if (autoGuessEncoding) { - encoding = detectEncoding(stdout, candidateGuessEncodings) || encoding; - } - - encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; - - return iconv.decode(stdout, encoding); - } - async buffer(object: string): Promise { const child = this.stream(['show', '--textconv', object]); diff --git a/extensions/git/src/repository.ts b/extensions/git/src/repository.ts index ca8974f2fc5..2f164ee5ede 100644 --- a/extensions/git/src/repository.ts +++ b/extensions/git/src/repository.ts @@ -7,7 +7,6 @@ import TelemetryReporter from '@vscode/extension-telemetry'; import * as fs from 'fs'; import * as path from 'path'; import picomatch from 'picomatch'; -import * as iconv from '@vscode/iconv-lite-umd'; import { CancellationError, CancellationToken, CancellationTokenSource, Command, commands, Disposable, Event, EventEmitter, FileDecoration, l10n, LogLevel, LogOutputChannel, Memento, ProgressLocation, ProgressOptions, QuickDiffProvider, RelativePattern, scm, SourceControl, SourceControlInputBox, SourceControlInputBoxValidation, SourceControlInputBoxValidationType, SourceControlResourceDecorations, SourceControlResourceGroup, SourceControlResourceState, TabInputNotebookDiff, TabInputTextDiff, TabInputTextMultiDiff, ThemeColor, Uri, window, workspace, WorkspaceEdit } from 'vscode'; import { ActionButton } from './actionButton'; import { ApiRepository } from './api/api1'; @@ -25,7 +24,6 @@ import { StatusBarCommands } from './statusbar'; import { toGitUri } from './uri'; import { anyEvent, combinedDisposable, debounceEvent, dispose, EmptyDisposable, eventToPromise, filterEvent, find, getCommitShortHash, IDisposable, isDescendant, isRemote, Limiter, onceEvent, pathEquals, relativePath } from './util'; import { IFileWatcher, watch } from './watch'; -import { detectEncoding } from './encoding'; import { ISourceControlHistoryItemDetailsProviderRegistry } from './historyItemDetailsProvider'; const timeout = (millis: number) => new Promise(c => setTimeout(c, millis)); @@ -1222,19 +1220,9 @@ export class Repository implements Disposable { await this.run(Operation.Remove, () => this.repository.rm(resources.map(r => r.fsPath))); } - async stage(resource: Uri, contents: string): Promise { - const path = relativePath(this.repository.root, resource.fsPath).replace(/\\/g, '/'); + async stage(resource: Uri, contents: string, encoding: string): Promise { await this.run(Operation.Stage, async () => { - const configFiles = workspace.getConfiguration('files', Uri.file(resource.fsPath)); - let encoding = configFiles.get('encoding') ?? 'utf8'; - const autoGuessEncoding = configFiles.get('autoGuessEncoding') === true; - const candidateGuessEncodings = configFiles.get('candidateGuessEncodings') ?? []; - - if (autoGuessEncoding) { - encoding = detectEncoding(Buffer.from(contents), candidateGuessEncodings) ?? encoding; - } - - encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; + const path = relativePath(this.repository.root, resource.fsPath).replace(/\\/g, '/'); await this.repository.stage(path, contents, encoding); this._onDidChangeOriginalResource.fire(resource); @@ -1980,17 +1968,15 @@ export class Repository implements Disposable { async show(ref: string, filePath: string): Promise { return await this.run(Operation.Show, async () => { const path = relativePath(this.repository.root, filePath).replace(/\\/g, '/'); - const configFiles = workspace.getConfiguration('files', Uri.file(filePath)); - const defaultEncoding = configFiles.get('encoding'); - const autoGuessEncoding = configFiles.get('autoGuessEncoding'); - const candidateGuessEncodings = configFiles.get('candidateGuessEncodings'); try { - return await this.repository.bufferString(`${ref}:${path}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); + const content = await this.repository.buffer(`${ref}:${path}`); + return await workspace.decode(content, Uri.file(filePath)); } catch (err) { if (err.gitErrorCode === GitErrorCodes.WrongCase) { const gitRelativePath = await this.repository.getGitRelativePath(ref, path); - return await this.repository.bufferString(`${ref}:${gitRelativePath}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); + const content = await this.repository.buffer(`${ref}:${gitRelativePath}`); + return await workspace.decode(content, Uri.file(filePath)); } throw err; diff --git a/extensions/git/tsconfig.json b/extensions/git/tsconfig.json index 42218d8decb..1d330ea2516 100644 --- a/extensions/git/tsconfig.json +++ b/extensions/git/tsconfig.json @@ -25,6 +25,7 @@ "../../src/vscode-dts/vscode.proposed.statusBarItemTooltip.d.ts", "../../src/vscode-dts/vscode.proposed.tabInputMultiDiff.d.ts", "../../src/vscode-dts/vscode.proposed.tabInputTextMerge.d.ts", + "../../src/vscode-dts/vscode.proposed.textDocumentEncoding.d.ts", "../../src/vscode-dts/vscode.proposed.textEditorDiffInformation.d.ts", "../../src/vscode-dts/vscode.proposed.timeline.d.ts", "../types/lib.textEncoder.d.ts"