diff --git a/extensions/git/package-lock.json b/extensions/git/package-lock.json index bc150555c70..2629a940444 100644 --- a/extensions/git/package-lock.json +++ b/extensions/git/package-lock.json @@ -11,10 +11,8 @@ "dependencies": { "@joaomoreno/unique-names-generator": "^5.2.0", "@vscode/extension-telemetry": "^0.9.8", - "@vscode/iconv-lite-umd": "0.7.0", "byline": "^5.0.0", "file-type": "16.5.4", - "jschardet": "3.1.4", "picomatch": "2.3.1", "vscode-uri": "^2.0.0", "which": "4.0.0" @@ -218,11 +216,6 @@ "vscode": "^1.75.0" } }, - "node_modules/@vscode/iconv-lite-umd": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@vscode/iconv-lite-umd/-/iconv-lite-umd-0.7.0.tgz", - "integrity": "sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg==" - }, "node_modules/byline": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", @@ -279,15 +272,6 @@ "node": ">=16" } }, - "node_modules/jschardet": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/jschardet/-/jschardet-3.1.4.tgz", - "integrity": "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg==", - "license": "LGPL-2.1+", - "engines": { - "node": ">=0.1.90" - } - }, "node_modules/peek-readable": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", diff --git a/extensions/git/package.json b/extensions/git/package.json index 0967a02f71d..c46a736fe00 100644 --- a/extensions/git/package.json +++ b/extensions/git/package.json @@ -3567,10 +3567,8 @@ "dependencies": { "@joaomoreno/unique-names-generator": "^5.2.0", "@vscode/extension-telemetry": "^0.9.8", - "@vscode/iconv-lite-umd": "0.7.0", "byline": "^5.0.0", "file-type": "16.5.4", - "jschardet": "3.1.4", "picomatch": "2.3.1", "vscode-uri": "^2.0.0", "which": "4.0.0" diff --git a/extensions/git/src/commands.ts b/extensions/git/src/commands.ts index 1094a438b07..107c953d394 100644 --- a/extensions/git/src/commands.ts +++ b/extensions/git/src/commands.ts @@ -1636,19 +1636,28 @@ export class CommandCenter { } let modifiedUri = changes.modifiedUri; + let modifiedDocument: TextDocument | undefined; + if (!modifiedUri) { const textEditor = window.activeTextEditor; if (!textEditor) { return; } - const modifiedDocument = textEditor.document; + modifiedDocument = textEditor.document; modifiedUri = modifiedDocument.uri; } + if (modifiedUri.scheme !== 'file') { return; } + + if (!modifiedDocument) { + modifiedDocument = await workspace.openTextDocument(modifiedUri); + } + const result = changes.originalWithModifiedChanges; - await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result)); + await this.runByRepository(modifiedUri, async (repository, resource) => + await repository.stage(resource, result, modifiedDocument.encoding)); } @command('git.stageSelectedRanges') @@ -1824,7 +1833,8 @@ export class CommandCenter { const originalDocument = await workspace.openTextDocument(originalUri); const result = applyLineChanges(originalDocument, modifiedDocument, changes); - await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result)); + await this.runByRepository(modifiedUri, async (repository, resource) => + await repository.stage(resource, result, modifiedDocument.encoding)); } @command('git.revertChange') @@ -1994,7 +2004,7 @@ export class CommandCenter { this.logger.trace(`[CommandCenter][unstageSelectedRanges] invertedDiffs: ${JSON.stringify(invertedDiffs)}`); const result = applyLineChanges(modifiedDocument, originalDocument, invertedDiffs); - await repository.stage(modifiedUri, result); + await repository.stage(modifiedUri, result, modifiedDocument.encoding); } @command('git.unstageFile') diff --git a/extensions/git/src/encoding.ts b/extensions/git/src/encoding.ts deleted file mode 100644 index c80fb6ee6d5..00000000000 --- a/extensions/git/src/encoding.ts +++ /dev/null @@ -1,100 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as jschardet from 'jschardet'; - -function detectEncodingByBOM(buffer: Buffer): string | null { - if (!buffer || buffer.length < 2) { - return null; - } - - const b0 = buffer.readUInt8(0); - const b1 = buffer.readUInt8(1); - - // UTF-16 BE - if (b0 === 0xFE && b1 === 0xFF) { - return 'utf16be'; - } - - // UTF-16 LE - if (b0 === 0xFF && b1 === 0xFE) { - return 'utf16le'; - } - - if (buffer.length < 3) { - return null; - } - - const b2 = buffer.readUInt8(2); - - // UTF-8 - if (b0 === 0xEF && b1 === 0xBB && b2 === 0xBF) { - return 'utf8'; - } - - return null; -} - -const IGNORE_ENCODINGS = [ - 'ascii', - 'utf-8', - 'utf-16', - 'utf-32' -]; - -const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = { - 'ibm866': 'cp866', - 'big5': 'cp950' -}; - -const MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET: { [key: string]: string } = { - utf8: 'UTF-8', - utf16le: 'UTF-16LE', - utf16be: 'UTF-16BE', - windows1252: 'windows-1252', - windows1250: 'windows-1250', - iso88592: 'ISO-8859-2', - windows1251: 'windows-1251', - cp866: 'IBM866', - iso88595: 'ISO-8859-5', - koi8r: 'KOI8-R', - windows1253: 'windows-1253', - iso88597: 'ISO-8859-7', - windows1255: 'windows-1255', - iso88598: 'ISO-8859-8', - cp950: 'Big5', - shiftjis: 'SHIFT_JIS', - eucjp: 'EUC-JP', - euckr: 'EUC-KR', - gb2312: 'GB2312' -}; - -export function detectEncoding(buffer: Buffer, candidateGuessEncodings: string[]): string | null { - const result = detectEncodingByBOM(buffer); - - if (result) { - return result; - } - - candidateGuessEncodings = candidateGuessEncodings.map(e => MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET[e]).filter(e => !!e); - - const detected = jschardet.detect(buffer, candidateGuessEncodings.length > 0 ? { detectEncodings: candidateGuessEncodings } : undefined); - if (!detected || !detected.encoding) { - return null; - } - - const encoding = detected.encoding; - - // Ignore encodings that cannot guess correctly - // (http://chardet.readthedocs.io/en/latest/supported-encodings.html) - if (0 <= IGNORE_ENCODINGS.indexOf(encoding.toLowerCase())) { - return null; - } - - const normalizedEncodingName = encoding.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(); - const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName]; - - return mapped || normalizedEncodingName; -} diff --git a/extensions/git/src/git.ts b/extensions/git/src/git.ts index 4f64607aa6a..e23967252f5 100644 --- a/extensions/git/src/git.ts +++ b/extensions/git/src/git.ts @@ -14,7 +14,6 @@ import * as iconv from '@vscode/iconv-lite-umd'; import * as filetype from 'file-type'; import { assign, groupBy, IDisposable, toDisposable, dispose, mkdirp, readBytes, detectUnicodeEncoding, Encoding, onceEvent, splitInChunks, Limiter, Versions, isWindows, pathEquals, isMacintosh, isDescendant, relativePath } from './util'; import { CancellationError, CancellationToken, ConfigurationChangeEvent, LogOutputChannel, Progress, Uri, workspace } from 'vscode'; -import { detectEncoding } from './encoding'; import { Commit as ApiCommit, Ref, RefType, Branch, Remote, ForcePushMode, GitErrorCodes, LogOptions, Change, Status, CommitOptions, RefQuery, InitOptions } from './api/git'; import * as byline from 'byline'; import { StringDecoder } from 'string_decoder'; @@ -194,7 +193,6 @@ function cpErrorHandler(cb: (reason?: any) => void): (reason?: any) => void { export interface SpawnOptions extends cp.SpawnOptions { input?: string; - encoding?: string; log?: boolean; cancellationToken?: CancellationToken; onSpawn?: (childProcess: cp.ChildProcess) => void; @@ -621,12 +619,9 @@ export class Git { } } - let encoding = options.encoding || 'utf8'; - encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; - const result: IExecutionResult = { exitCode: bufferResult.exitCode, - stdout: iconv.decode(bufferResult.stdout, encoding), + stdout: bufferResult.stdout.toString('utf8'), stderr: bufferResult.stderr }; @@ -1398,18 +1393,6 @@ export class Repository { .filter(entry => !!entry); } - async bufferString(ref: string, filePath: string, encoding: string = 'utf8', autoGuessEncoding = false, candidateGuessEncodings: string[] = []): Promise { - const stdout = await this.buffer(ref, filePath); - - if (autoGuessEncoding) { - encoding = detectEncoding(stdout, candidateGuessEncodings) || encoding; - } - - encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; - - return iconv.decode(stdout, encoding); - } - async buffer(ref: string, filePath: string): Promise { const relativePath = sanitizeRelativePath(this.repositoryRoot, filePath); const child = this.stream(['show', '--textconv', `${ref}:${relativePath}`]); diff --git a/extensions/git/src/repository.ts b/extensions/git/src/repository.ts index f32d915d715..ae5c65040b9 100644 --- a/extensions/git/src/repository.ts +++ b/extensions/git/src/repository.ts @@ -7,7 +7,6 @@ import TelemetryReporter from '@vscode/extension-telemetry'; import * as fs from 'fs'; import * as path from 'path'; import picomatch from 'picomatch'; -import * as iconv from '@vscode/iconv-lite-umd'; import { CancellationError, CancellationToken, CancellationTokenSource, Command, commands, Disposable, Event, EventEmitter, FileDecoration, l10n, LogLevel, LogOutputChannel, Memento, ProgressLocation, ProgressOptions, QuickDiffProvider, RelativePattern, scm, SourceControl, SourceControlInputBox, SourceControlInputBoxValidation, SourceControlInputBoxValidationType, SourceControlResourceDecorations, SourceControlResourceGroup, SourceControlResourceState, TabInputNotebookDiff, TabInputTextDiff, TabInputTextMultiDiff, ThemeColor, Uri, window, workspace, WorkspaceEdit } from 'vscode'; import { ActionButton } from './actionButton'; import { ApiRepository } from './api/api1'; @@ -25,7 +24,6 @@ import { StatusBarCommands } from './statusbar'; import { toGitUri } from './uri'; import { anyEvent, combinedDisposable, debounceEvent, dispose, EmptyDisposable, eventToPromise, filterEvent, find, getCommitShortHash, IDisposable, isDescendant, isLinuxSnap, isRemote, Limiter, onceEvent, pathEquals, relativePath } from './util'; import { IFileWatcher, watch } from './watch'; -import { detectEncoding } from './encoding'; import { ISourceControlHistoryItemDetailsProviderRegistry } from './historyItemDetailsProvider'; const timeout = (millis: number) => new Promise(c => setTimeout(c, millis)); @@ -1222,18 +1220,8 @@ export class Repository implements Disposable { await this.run(Operation.Remove, () => this.repository.rm(resources.map(r => r.fsPath))); } - async stage(resource: Uri, contents: string): Promise { + async stage(resource: Uri, contents: string, encoding: string): Promise { await this.run(Operation.Stage, async () => { - const configFiles = workspace.getConfiguration('files', Uri.file(resource.fsPath)); - let encoding = configFiles.get('encoding') ?? 'utf8'; - const autoGuessEncoding = configFiles.get('autoGuessEncoding') === true; - const candidateGuessEncodings = configFiles.get('candidateGuessEncodings') ?? []; - - if (autoGuessEncoding) { - encoding = detectEncoding(Buffer.from(contents), candidateGuessEncodings) ?? encoding; - } - - encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; await this.repository.stage(resource.fsPath, contents, encoding); this._onDidChangeOriginalResource.fire(resource); @@ -1976,17 +1964,14 @@ export class Repository implements Disposable { async show(ref: string, filePath: string): Promise { return await this.run(Operation.Show, async () => { - const configFiles = workspace.getConfiguration('files', Uri.file(filePath)); - const defaultEncoding = configFiles.get('encoding'); - const autoGuessEncoding = configFiles.get('autoGuessEncoding'); - const candidateGuessEncodings = configFiles.get('candidateGuessEncodings'); - try { - return await this.repository.bufferString(ref, filePath, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); + const content = await this.repository.buffer(ref, filePath); + return await workspace.decode(content, Uri.file(filePath)); } catch (err) { if (err.gitErrorCode === GitErrorCodes.WrongCase) { const gitFilePath = await this.repository.getGitFilePath(ref, filePath); - return await this.repository.bufferString(ref, gitFilePath, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); + const content = await this.repository.buffer(ref, gitFilePath); + return await workspace.decode(content, Uri.file(filePath)); } throw err; diff --git a/extensions/git/tsconfig.json b/extensions/git/tsconfig.json index c79be520be9..fc72bd70742 100644 --- a/extensions/git/tsconfig.json +++ b/extensions/git/tsconfig.json @@ -24,6 +24,7 @@ "../../src/vscode-dts/vscode.proposed.statusBarItemTooltip.d.ts", "../../src/vscode-dts/vscode.proposed.tabInputMultiDiff.d.ts", "../../src/vscode-dts/vscode.proposed.tabInputTextMerge.d.ts", + "../../src/vscode-dts/vscode.proposed.textDocumentEncoding.d.ts", "../../src/vscode-dts/vscode.proposed.textEditorDiffInformation.d.ts", "../../src/vscode-dts/vscode.proposed.timeline.d.ts", "../types/lib.textEncoder.d.ts"