diff --git a/extensions/git/src/encoding.ts b/extensions/git/src/encoding.ts index a283f628594..c80fb6ee6d5 100644 --- a/extensions/git/src/encoding.ts +++ b/extensions/git/src/encoding.ts @@ -49,15 +49,38 @@ const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = { 'big5': 'cp950' }; -export function detectEncoding(buffer: Buffer): string | null { +const MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET: { [key: string]: string } = { + utf8: 'UTF-8', + utf16le: 'UTF-16LE', + utf16be: 'UTF-16BE', + windows1252: 'windows-1252', + windows1250: 'windows-1250', + iso88592: 'ISO-8859-2', + windows1251: 'windows-1251', + cp866: 'IBM866', + iso88595: 'ISO-8859-5', + koi8r: 'KOI8-R', + windows1253: 'windows-1253', + iso88597: 'ISO-8859-7', + windows1255: 'windows-1255', + iso88598: 'ISO-8859-8', + cp950: 'Big5', + shiftjis: 'SHIFT_JIS', + eucjp: 'EUC-JP', + euckr: 'EUC-KR', + gb2312: 'GB2312' +}; + +export function detectEncoding(buffer: Buffer, candidateGuessEncodings: string[]): string | null { const result = detectEncodingByBOM(buffer); if (result) { return result; } - const detected = jschardet.detect(buffer); + candidateGuessEncodings = candidateGuessEncodings.map(e => MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET[e]).filter(e => !!e); + const detected = jschardet.detect(buffer, candidateGuessEncodings.length > 0 ? { detectEncodings: candidateGuessEncodings } : undefined); if (!detected || !detected.encoding) { return null; } diff --git a/extensions/git/src/git.ts b/extensions/git/src/git.ts index 697e77815e4..46fcd6069a6 100644 --- a/extensions/git/src/git.ts +++ b/extensions/git/src/git.ts @@ -1233,11 +1233,11 @@ export class Repository { .filter(entry => !!entry); } - async bufferString(object: string, encoding: string = 'utf8', autoGuessEncoding = false): Promise { + async bufferString(object: string, encoding: string = 'utf8', autoGuessEncoding = false, candidateGuessEncodings: string[] = []): Promise { const stdout = await this.buffer(object); if (autoGuessEncoding) { - encoding = detectEncoding(stdout) || encoding; + encoding = detectEncoding(stdout, candidateGuessEncodings) || encoding; } encoding = iconv.encodingExists(encoding) ? encoding : 'utf8'; diff --git a/extensions/git/src/repository.ts b/extensions/git/src/repository.ts index ed959765a59..3863e748726 100644 --- a/extensions/git/src/repository.ts +++ b/extensions/git/src/repository.ts @@ -1865,13 +1865,14 @@ export class Repository implements Disposable { const configFiles = workspace.getConfiguration('files', Uri.file(filePath)); const defaultEncoding = configFiles.get('encoding'); const autoGuessEncoding = configFiles.get('autoGuessEncoding'); + const candidateGuessEncodings = configFiles.get('candidateGuessEncodings'); try { - return await this.repository.bufferString(`${ref}:${path}`, defaultEncoding, autoGuessEncoding); + return await this.repository.bufferString(`${ref}:${path}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); } catch (err) { if (err.gitErrorCode === GitErrorCodes.WrongCase) { const gitRelativePath = await this.repository.getGitRelativePath(ref, path); - return await this.repository.bufferString(`${ref}:${gitRelativePath}`, defaultEncoding, autoGuessEncoding); + return await this.repository.bufferString(`${ref}:${gitRelativePath}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings); } throw err; diff --git a/src/vs/platform/files/common/files.ts b/src/vs/platform/files/common/files.ts index 149665bb571..290a991a35d 100644 --- a/src/vs/platform/files/common/files.ts +++ b/src/vs/platform/files/common/files.ts @@ -1470,6 +1470,7 @@ export interface IFilesConfigurationNode { watcherInclude: string[]; encoding: string; autoGuessEncoding: boolean; + candidateGuessEncodings: string[]; defaultLanguage: string; trimTrailingWhitespace: boolean; autoSave: string; diff --git a/src/vs/workbench/browser/parts/editor/editorStatus.ts b/src/vs/workbench/browser/parts/editor/editorStatus.ts index 83016b30a46..845160784b1 100644 --- a/src/vs/workbench/browser/parts/editor/editorStatus.ts +++ b/src/vs/workbench/browser/parts/editor/editorStatus.ts @@ -1453,13 +1453,16 @@ export class ChangeEncodingAction extends Action2 { let guessedEncoding: string | undefined = undefined; if (fileService.hasProvider(resource)) { - const content = await textFileService.readStream(resource, { autoGuessEncoding: true }); + const content = await textFileService.readStream(resource, { + autoGuessEncoding: true, + candidateGuessEncodings: textResourceConfigurationService.getValue(resource, 'files.candidateGuessEncodings') + }); guessedEncoding = content.encoding; } const isReopenWithEncoding = (action === reopenWithEncodingPick); - const configuredEncoding = textResourceConfigurationService.getValue(resource ?? undefined, 'files.encoding'); + const configuredEncoding = textResourceConfigurationService.getValue(resource, 'files.encoding'); let directMatchIndex: number | undefined; let aliasMatchIndex: number | undefined; diff --git a/src/vs/workbench/contrib/files/browser/files.contribution.ts b/src/vs/workbench/contrib/files/browser/files.contribution.ts index 35a5a532e0f..d525ba5860c 100644 --- a/src/vs/workbench/contrib/files/browser/files.contribution.ts +++ b/src/vs/workbench/contrib/files/browser/files.contribution.ts @@ -23,7 +23,7 @@ import { IEditorPaneRegistry, EditorPaneDescriptor } from 'vs/workbench/browser/ import { ILabelService } from 'vs/platform/label/common/label'; import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions'; import { ExplorerService, UNDO_REDO_SOURCE } from 'vs/workbench/contrib/files/browser/explorerService'; -import { SUPPORTED_ENCODINGS } from 'vs/workbench/services/textfile/common/encoding'; +import { GUESSABLE_ENCODINGS, SUPPORTED_ENCODINGS } from 'vs/workbench/services/textfile/common/encoding'; import { Schemas } from 'vs/base/common/network'; import { WorkspaceWatcher } from 'vs/workbench/contrib/files/browser/workspaceWatcher'; import { editorConfigurationBaseNode } from 'vs/editor/common/config/editorConfigurationSchema'; @@ -202,6 +202,17 @@ configurationRegistry.registerConfiguration({ 'markdownDescription': nls.localize('autoGuessEncoding', "When enabled, the editor will attempt to guess the character set encoding when opening files. This setting can also be configured per language. Note, this setting is not respected by text search. Only {0} is respected.", '`#files.encoding#`'), 'scope': ConfigurationScope.LANGUAGE_OVERRIDABLE }, + 'files.candidateGuessEncodings': { + 'type': 'array', + 'items': { + 'type': 'string', + 'enum': Object.keys(GUESSABLE_ENCODINGS), + 'enumDescriptions': Object.keys(GUESSABLE_ENCODINGS).map(key => GUESSABLE_ENCODINGS[key].labelLong) + }, + 'default': [], + 'markdownDescription': nls.localize('candidateGuessEncodings', "List of character set encodings that the editor should attempt to guess in the order they are listed. In case it cannot be determined, {0} is respected", '`#files.encoding#`'), + 'scope': ConfigurationScope.LANGUAGE_OVERRIDABLE + }, 'files.eol': { 'type': 'string', 'enum': [ diff --git a/src/vs/workbench/services/textfile/browser/textFileService.ts b/src/vs/workbench/services/textfile/browser/textFileService.ts index b388e7cc334..859fa8a1c1a 100644 --- a/src/vs/workbench/services/textfile/browser/textFileService.ts +++ b/src/vs/workbench/services/textfile/browser/textFileService.ts @@ -304,7 +304,12 @@ export abstract class AbstractTextFileService extends Disposable implements ITex // read through encoding library return toDecodeStream(stream, { acceptTextOnly: options?.acceptTextOnly ?? false, - guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'), + guessEncoding: + options?.autoGuessEncoding || + this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'), + candidateGuessEncodings: + options?.candidateGuessEncodings || + this.textResourceConfigurationService.getValue(resource, 'files.candidateGuessEncodings'), overwriteEncoding: async detectedEncoding => { const { encoding } = await this.encoding.getPreferredReadEncoding(resource, options, detectedEncoding ?? undefined); diff --git a/src/vs/workbench/services/textfile/common/encoding.ts b/src/vs/workbench/services/textfile/common/encoding.ts index ad67fb4f422..bae27136de4 100644 --- a/src/vs/workbench/services/textfile/common/encoding.ts +++ b/src/vs/workbench/services/textfile/common/encoding.ts @@ -7,6 +7,7 @@ import { Readable, ReadableStream, newWriteableStream, listenStream } from 'vs/b import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer'; import { importAMDNodeModule } from 'vs/amdX'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; +import { coalesce } from 'vs/base/common/arrays'; export const UTF8 = 'utf8'; export const UTF8_with_bom = 'utf8bom'; @@ -31,6 +32,7 @@ const AUTO_ENCODING_GUESS_MAX_BYTES = 512 * 128; // set an upper limit for the export interface IDecodeStreamOptions { acceptTextOnly: boolean; guessEncoding: boolean; + candidateGuessEncodings: string[]; minBytesRequiredForDetection?: number; overwriteEncoding(detectedEncoding: string | null): Promise; @@ -134,7 +136,7 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS const detected = await detectEncodingFromBuffer({ buffer: VSBuffer.concat(bufferedChunks), bytesRead: bytesBuffered - }, options.guessEncoding); + }, options.guessEncoding, options.candidateGuessEncodings); // throw early if the source seems binary and // we are instructed to only accept text @@ -317,7 +319,7 @@ const IGNORE_ENCODINGS = ['ascii', 'utf-16', 'utf-32']; /** * Guesses the encoding from buffer. */ -async function guessEncodingByBuffer(buffer: VSBuffer): Promise { +async function guessEncodingByBuffer(buffer: VSBuffer, candidateGuessEncodings?: string[]): Promise { const jschardet = await importAMDNodeModule('jschardet', 'dist/jschardet.min.js'); // ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53 @@ -328,7 +330,15 @@ async function guessEncodingByBuffer(buffer: VSBuffer): Promise { // https://github.com/aadsm/jschardet/blob/v2.1.1/src/index.js#L36-L40 const binaryString = encodeLatin1(limitedBuffer.buffer); - const guessed = jschardet.detect(binaryString); + // ensure to convert candidate encodings to jschardet encoding names if provided + if (candidateGuessEncodings) { + candidateGuessEncodings = coalesce(candidateGuessEncodings.map(e => toJschardetEncoding(e))); + if (candidateGuessEncodings.length === 0) { + candidateGuessEncodings = undefined; + } + } + + const guessed = jschardet.detect(binaryString, candidateGuessEncodings ? { detectEncodings: candidateGuessEncodings } : undefined); if (!guessed || !guessed.encoding) { return null; } @@ -346,13 +356,24 @@ const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = { 'big5': 'cp950' }; +function normalizeEncoding(encodingName: string): string { + return encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(); +} + function toIconvLiteEncoding(encodingName: string): string { - const normalizedEncodingName = encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(); + const normalizedEncodingName = normalizeEncoding(encodingName); const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName]; return mapped || normalizedEncodingName; } +function toJschardetEncoding(encodingName: string): string | undefined { + const normalizedEncodingName = normalizeEncoding(encodingName); + const mapped = GUESSABLE_ENCODINGS[normalizedEncodingName]; + + return mapped.guessableName; +} + function encodeLatin1(buffer: Uint8Array): string { let result = ''; for (let i = 0; i < buffer.length; i++) { @@ -410,9 +431,9 @@ export interface IReadResult { bytesRead: number; } -export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: false): IDetectedEncodingResult; -export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: boolean): Promise; -export function detectEncodingFromBuffer({ buffer, bytesRead }: IReadResult, autoGuessEncoding?: boolean): Promise | IDetectedEncodingResult { +export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: false, candidateGuessEncodings?: string[]): IDetectedEncodingResult; +export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: boolean, candidateGuessEncodings?: string[]): Promise; +export function detectEncodingFromBuffer({ buffer, bytesRead }: IReadResult, autoGuessEncoding?: boolean, candidateGuessEncodings?: string[]): Promise | IDetectedEncodingResult { // Always first check for BOM to find out about encoding let encoding = detectEncodingByBOMFromBuffer(buffer, bytesRead); @@ -469,7 +490,7 @@ export function detectEncodingFromBuffer({ buffer, bytesRead }: IReadResult, aut // Auto guess encoding if configured if (autoGuessEncoding && !seemsBinary && !encoding && buffer) { - return guessEncodingByBuffer(buffer.slice(0, bytesRead)).then(guessedEncoding => { + return guessEncodingByBuffer(buffer.slice(0, bytesRead), candidateGuessEncodings).then(guessedEncoding => { return { seemsBinary: false, encoding: guessedEncoding @@ -480,12 +501,15 @@ export function detectEncodingFromBuffer({ buffer, bytesRead }: IReadResult, aut return { seemsBinary, encoding }; } -export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; labelShort: string; order: number; encodeOnly?: boolean; alias?: string } } = { +type EncodingsMap = { [encoding: string]: { labelLong: string; labelShort: string; order: number; encodeOnly?: boolean; alias?: string; guessableName?: string } }; + +export const SUPPORTED_ENCODINGS: EncodingsMap = { utf8: { labelLong: 'UTF-8', labelShort: 'UTF-8', order: 1, - alias: 'utf8bom' + alias: 'utf8bom', + guessableName: 'UTF-8' }, utf8bom: { labelLong: 'UTF-8 with BOM', @@ -497,17 +521,20 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab utf16le: { labelLong: 'UTF-16 LE', labelShort: 'UTF-16 LE', - order: 3 + order: 3, + guessableName: 'UTF-16LE' }, utf16be: { labelLong: 'UTF-16 BE', labelShort: 'UTF-16 BE', - order: 4 + order: 4, + guessableName: 'UTF-16BE' }, windows1252: { labelLong: 'Western (Windows 1252)', labelShort: 'Windows 1252', - order: 5 + order: 5, + guessableName: 'windows-1252' }, iso88591: { labelLong: 'Western (ISO 8859-1)', @@ -562,12 +589,14 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab windows1250: { labelLong: 'Central European (Windows 1250)', labelShort: 'Windows 1250', - order: 16 + order: 16, + guessableName: 'windows-1250' }, iso88592: { labelLong: 'Central European (ISO 8859-2)', labelShort: 'ISO 8859-2', - order: 17 + order: 17, + guessableName: 'ISO-8859-2' }, cp852: { labelLong: 'Central European (CP 852)', @@ -577,22 +606,26 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab windows1251: { labelLong: 'Cyrillic (Windows 1251)', labelShort: 'Windows 1251', - order: 19 + order: 19, + guessableName: 'windows-1251' }, cp866: { labelLong: 'Cyrillic (CP 866)', labelShort: 'CP 866', - order: 20 + order: 20, + guessableName: 'IBM866' }, iso88595: { labelLong: 'Cyrillic (ISO 8859-5)', labelShort: 'ISO 8859-5', - order: 21 + order: 21, + guessableName: 'ISO-8859-5' }, koi8r: { labelLong: 'Cyrillic (KOI8-R)', labelShort: 'KOI8-R', - order: 22 + order: 22, + guessableName: 'KOI8-R' }, koi8u: { labelLong: 'Cyrillic (KOI8-U)', @@ -607,22 +640,26 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab windows1253: { labelLong: 'Greek (Windows 1253)', labelShort: 'Windows 1253', - order: 25 + order: 25, + guessableName: 'windows-1253' }, iso88597: { labelLong: 'Greek (ISO 8859-7)', labelShort: 'ISO 8859-7', - order: 26 + order: 26, + guessableName: 'ISO-8859-7' }, windows1255: { labelLong: 'Hebrew (Windows 1255)', labelShort: 'Windows 1255', - order: 27 + order: 27, + guessableName: 'windows-1255' }, iso88598: { labelLong: 'Hebrew (ISO 8859-8)', labelShort: 'ISO 8859-8', - order: 28 + order: 28, + guessableName: 'ISO-8859-8' }, iso885910: { labelLong: 'Nordic (ISO 8859-10)', @@ -662,7 +699,8 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab cp950: { labelLong: 'Traditional Chinese (Big5)', labelShort: 'Big5', - order: 36 + order: 36, + guessableName: 'Big5' }, big5hkscs: { labelLong: 'Traditional Chinese (Big5-HKSCS)', @@ -672,17 +710,20 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab shiftjis: { labelLong: 'Japanese (Shift JIS)', labelShort: 'Shift JIS', - order: 38 + order: 38, + guessableName: 'SHIFT_JIS' }, eucjp: { labelLong: 'Japanese (EUC-JP)', labelShort: 'EUC-JP', - order: 39 + order: 39, + guessableName: 'EUC-JP' }, euckr: { labelLong: 'Korean (EUC-KR)', labelShort: 'EUC-KR', - order: 40 + order: 40, + guessableName: 'EUC-KR' }, windows874: { labelLong: 'Thai (Windows 874)', @@ -707,7 +748,8 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab gb2312: { labelLong: 'Simplified Chinese (GB 2312)', labelShort: 'GB 2312', - order: 45 + order: 45, + guessableName: 'GB2312' }, cp865: { labelLong: 'Nordic DOS (CP 865)', @@ -720,3 +762,14 @@ export const SUPPORTED_ENCODINGS: { [encoding: string]: { labelLong: string; lab order: 47 } }; + +export const GUESSABLE_ENCODINGS: EncodingsMap = (() => { + const guessableEncodings: EncodingsMap = {}; + for (const encoding in SUPPORTED_ENCODINGS) { + if (SUPPORTED_ENCODINGS[encoding].guessableName) { + guessableEncodings[encoding] = SUPPORTED_ENCODINGS[encoding]; + } + } + + return guessableEncodings; +})(); diff --git a/src/vs/workbench/services/textfile/common/textfiles.ts b/src/vs/workbench/services/textfile/common/textfiles.ts index 73abb110fb1..d33c93d5d54 100644 --- a/src/vs/workbench/services/textfile/common/textfiles.ts +++ b/src/vs/workbench/services/textfile/common/textfiles.ts @@ -131,6 +131,11 @@ export interface IReadTextFileEncodingOptions { */ readonly autoGuessEncoding?: boolean; + /** + * The optional candidateGuessEncodings parameter limits the allowed encodings to guess from. + */ + readonly candidateGuessEncodings?: string[]; + /** * The optional acceptTextOnly parameter allows to fail this request early if the file * contents are not textual. diff --git a/src/vs/workbench/services/textfile/test/common/fixtures/files.ts b/src/vs/workbench/services/textfile/test/common/fixtures/files.ts index f3d0c5d6cd2..3167d521985 100644 --- a/src/vs/workbench/services/textfile/test/common/fixtures/files.ts +++ b/src/vs/workbench/services/textfile/test/common/fixtures/files.ts @@ -68,6 +68,9 @@ fixtures['index.html'] = Uint8Array.from([ 60, 33, 68, 79, 67, 84, 89, 80, 69, 32, 104, 116, 109, 108, 62, 10, 60, 104, 116, 109, 108, 62, 10, 60, 104, 101, 97, 100, 32, 105, 100, 61, 39, 104, 101, 97, 100, 73, 68, 39, 62, 10, 32, 32, 32, 32, 60, 109, 101, 116, 97, 32, 104, 116, 116, 112, 45, 101, 113, 117, 105, 118, 61, 34, 88, 45, 85, 65, 45, 67, 111, 109, 112, 97, 116, 105, 98, 108, 101, 34, 32, 99, 111, 110, 116, 101, 110, 116, 61, 34, 73, 69, 61, 101, 100, 103, 101, 34, 32, 47, 62, 10, 32, 32, 32, 32, 60, 116, 105, 116, 108, 101, 62, 83, 116, 114, 97, 100, 97, 32, 60, 47, 116, 105, 116, 108, 101, 62, 10, 32, 32, 32, 32, 60, 108, 105, 110, 107, 32, 104, 114, 101, 102, 61, 34, 115, 105, 116, 101, 46, 99, 115, 115, 34, 32, 114, 101, 108, 61, 34, 115, 116, 121, 108, 101, 115, 104, 101, 101, 116, 34, 32, 116, 121, 112, 101, 61, 34, 116, 101, 120, 116, 47, 99, 115, 115, 34, 32, 47, 62, 10, 32, 32, 32, 32, 60, 115, 99, 114, 105, 112, 116, 32, 115, 114, 99, 61, 34, 106, 113, 117, 101, 114, 121, 45, 49, 46, 52, 46, 49, 46, 106, 115, 34, 62, 60, 47, 115, 99, 114, 105, 112, 116, 62, 10, 32, 32, 32, 32, 60, 115, 99, 114, 105, 112, 116, 32, 115, 114, 99, 61, 34, 46, 46, 47, 99, 111, 109, 112, 105, 108, 101, 114, 47, 100, 116, 114, 101, 101, 46, 106, 115, 34, 32, 116, 121, 112, 101, 61, 34, 116, 101, 120, 116, 47, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 34, 62, 60, 47, 115, 99, 114, 105, 112, 116, 62, 10, 32, 32, 32, 32, 60, 115, 99, 114, 105, 112, 116, 32, 115, 114, 99, 61, 34, 46, 46, 47, 99, 111, 109, 112, 105, 108, 101, 114, 47, 116, 121, 112, 101, 115, 99, 114, 105, 112, 116, 46, 106, 115, 34, 32, 116, 121, 112, 101, 61, 34, 116, 101, 120, 116, 47, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 34, 62, 60, 47, 115, 99, 114, 105, 112, 116, 62, 10, 32, 32, 32, 32, 60, 115, 99, 114, 105, 112, 116, 32, 116, 121, 112, 101, 61, 34, 116, 101, 120, 116, 47, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 34, 62, 10, 10, 32, 32, 32, 32, 47, 47, 32, 67, 111, 109, 112, 105, 108, 101, 32, 115, 116, 114, 97, 100, 97, 32, 115, 111, 117, 114, 99, 101, 32, 105, 110, 116, 111, 32, 114, 101, 115, 117, 108, 116, 105, 110, 103, 32, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 10, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 99, 111, 109, 112, 105, 108, 101, 40, 112, 114, 111, 103, 44, 32, 108, 105, 98, 84, 101, 120, 116, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 111, 117, 116, 102, 105, 108, 101, 32, 61, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 115, 111, 117, 114, 99, 101, 58, 32, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 87, 114, 105, 116, 101, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 40, 115, 41, 32, 123, 32, 116, 104, 105, 115, 46, 115, 111, 117, 114, 99, 101, 32, 43, 61, 32, 115, 59, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 87, 114, 105, 116, 101, 76, 105, 110, 101, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 40, 115, 41, 32, 123, 32, 116, 104, 105, 115, 46, 115, 111, 117, 114, 99, 101, 32, 43, 61, 32, 115, 32, 43, 32, 34, 92, 114, 34, 59, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 32, 61, 32, 91, 93, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 99, 111, 109, 112, 105, 108, 101, 114, 61, 110, 101, 119, 32, 84, 111, 111, 108, 115, 46, 84, 121, 112, 101, 83, 99, 114, 105, 112, 116, 67, 111, 109, 112, 105, 108, 101, 114, 40, 111, 117, 116, 102, 105, 108, 101, 44, 116, 114, 117, 101, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 114, 46, 115, 101, 116, 69, 114, 114, 111, 114, 67, 97, 108, 108, 98, 97, 99, 107, 40, 102, 117, 110, 99, 116, 105, 111, 110, 40, 115, 116, 97, 114, 116, 44, 108, 101, 110, 44, 32, 109, 101, 115, 115, 97, 103, 101, 41, 32, 123, 32, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 46, 112, 117, 115, 104, 40, 123, 115, 116, 97, 114, 116, 58, 115, 116, 97, 114, 116, 44, 32, 108, 101, 110, 58, 108, 101, 110, 44, 32, 109, 101, 115, 115, 97, 103, 101, 58, 109, 101, 115, 115, 97, 103, 101, 125, 41, 59, 32, 125, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 114, 46, 97, 100, 100, 85, 110, 105, 116, 40, 108, 105, 98, 84, 101, 120, 116, 44, 34, 108, 105, 98, 46, 116, 115, 34, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 114, 46, 97, 100, 100, 85, 110, 105, 116, 40, 112, 114, 111, 103, 44, 34, 105, 110, 112, 117, 116, 46, 116, 115, 34, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 114, 46, 116, 121, 112, 101, 67, 104, 101, 99, 107, 40, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 114, 46, 101, 109, 105, 116, 40, 41, 59, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 105, 102, 40, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 46, 108, 101, 110, 103, 116, 104, 32, 62, 32, 48, 32, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 47, 47, 116, 104, 114, 111, 119, 32, 110, 101, 119, 32, 69, 114, 114, 111, 114, 40, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 9, 119, 104, 105, 108, 101, 40, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 91, 48, 93, 32, 61, 61, 32, 39, 47, 39, 32, 38, 38, 32, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 91, 49, 93, 32, 61, 61, 32, 39, 47, 39, 32, 38, 38, 32, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 91, 50, 93, 32, 61, 61, 32, 39, 32, 39, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 32, 61, 32, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 46, 115, 108, 105, 99, 101, 40, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 46, 105, 110, 100, 101, 120, 79, 102, 40, 39, 92, 114, 39, 41, 43, 49, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 101, 114, 114, 111, 114, 80, 114, 101, 102, 105, 120, 32, 61, 32, 34, 34, 59, 10, 9, 102, 111, 114, 40, 118, 97, 114, 32, 105, 32, 61, 32, 48, 59, 105, 60, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 46, 108, 101, 110, 103, 116, 104, 59, 105, 43, 43, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 101, 114, 114, 111, 114, 80, 114, 101, 102, 105, 120, 32, 43, 61, 32, 34, 47, 47, 32, 69, 114, 114, 111, 114, 58, 32, 40, 34, 32, 43, 32, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 91, 105, 93, 46, 115, 116, 97, 114, 116, 32, 43, 32, 34, 44, 34, 32, 43, 32, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 91, 105, 93, 46, 108, 101, 110, 32, 43, 32, 34, 41, 32, 34, 32, 43, 32, 112, 97, 114, 115, 101, 69, 114, 114, 111, 114, 115, 91, 105, 93, 46, 109, 101, 115, 115, 97, 103, 101, 32, 43, 32, 34, 92, 114, 34, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 114, 101, 116, 117, 114, 110, 32, 101, 114, 114, 111, 114, 80, 114, 101, 102, 105, 120, 32, 43, 32, 111, 117, 116, 102, 105, 108, 101, 46, 115, 111, 117, 114, 99, 101, 59, 10, 32, 32, 32, 32, 125, 10, 32, 32, 32, 32, 60, 47, 115, 99, 114, 105, 112, 116, 62, 10, 32, 32, 32, 32, 60, 115, 99, 114, 105, 112, 116, 32, 116, 121, 112, 101, 61, 34, 116, 101, 120, 116, 47, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 34, 62, 10, 9, 10, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 108, 105, 98, 84, 101, 120, 116, 32, 61, 32, 34, 34, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 36, 46, 103, 101, 116, 40, 34, 46, 46, 47, 99, 111, 109, 112, 105, 108, 101, 114, 47, 108, 105, 98, 46, 116, 115, 34, 44, 32, 102, 117, 110, 99, 116, 105, 111, 110, 40, 110, 101, 119, 76, 105, 98, 84, 101, 120, 116, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 108, 105, 98, 84, 101, 120, 116, 32, 61, 32, 110, 101, 119, 76, 105, 98, 84, 101, 120, 116, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 41, 59, 9, 10, 32, 32, 32, 32, 32, 32, 32, 32, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 47, 47, 32, 101, 120, 101, 99, 117, 116, 101, 32, 116, 104, 101, 32, 106, 97, 118, 97, 115, 99, 114, 105, 112, 116, 32, 105, 110, 32, 116, 104, 101, 32, 99, 111, 109, 112, 105, 108, 101, 100, 79, 117, 116, 112, 117, 116, 32, 112, 97, 110, 101, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 101, 120, 101, 99, 117, 116, 101, 40, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 36, 40, 39, 35, 99, 111, 109, 112, 105, 108, 97, 116, 105, 111, 110, 39, 41, 46, 116, 101, 120, 116, 40, 34, 82, 117, 110, 110, 105, 110, 103, 46, 46, 46, 34, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 116, 120, 116, 32, 61, 32, 36, 40, 39, 35, 99, 111, 109, 112, 105, 108, 101, 100, 79, 117, 116, 112, 117, 116, 39, 41, 46, 118, 97, 108, 40, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 114, 101, 115, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 116, 114, 121, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 114, 101, 116, 32, 61, 32, 101, 118, 97, 108, 40, 116, 120, 116, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 114, 101, 115, 32, 61, 32, 34, 82, 97, 110, 32, 115, 117, 99, 99, 101, 115, 115, 102, 117, 108, 108, 121, 33, 34, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 32, 99, 97, 116, 99, 104, 40, 101, 41, 32, 123, 32, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 114, 101, 115, 32, 61, 32, 34, 69, 120, 99, 101, 112, 116, 105, 111, 110, 32, 116, 104, 114, 111, 119, 110, 58, 32, 34, 32, 43, 32, 101, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 36, 40, 39, 35, 99, 111, 109, 112, 105, 108, 97, 116, 105, 111, 110, 39, 41, 46, 116, 101, 120, 116, 40, 83, 116, 114, 105, 110, 103, 40, 114, 101, 115, 41, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 47, 47, 32, 114, 101, 99, 111, 109, 112, 105, 108, 101, 32, 116, 104, 101, 32, 115, 116, 114, 97, 100, 97, 83, 114, 99, 32, 97, 110, 100, 32, 112, 111, 112, 117, 108, 97, 116, 101, 32, 116, 104, 101, 32, 99, 111, 109, 112, 105, 108, 101, 100, 79, 117, 116, 112, 117, 116, 32, 112, 97, 110, 101, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 115, 114, 99, 85, 112, 100, 97, 116, 101, 100, 40, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 110, 101, 119, 84, 101, 120, 116, 32, 61, 32, 36, 40, 39, 35, 115, 116, 114, 97, 100, 97, 83, 114, 99, 39, 41, 46, 118, 97, 108, 40, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 99, 111, 109, 112, 105, 108, 101, 100, 83, 111, 117, 114, 99, 101, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 116, 114, 121, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 100, 83, 111, 117, 114, 99, 101, 32, 61, 32, 99, 111, 109, 112, 105, 108, 101, 40, 110, 101, 119, 84, 101, 120, 116, 44, 32, 108, 105, 98, 84, 101, 120, 116, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 32, 99, 97, 116, 99, 104, 32, 40, 101, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 100, 83, 111, 117, 114, 99, 101, 32, 61, 32, 34, 47, 47, 80, 97, 114, 115, 101, 32, 101, 114, 114, 111, 114, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 102, 111, 114, 40, 118, 97, 114, 32, 105, 32, 105, 110, 32, 101, 41, 32, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 105, 108, 101, 100, 83, 111, 117, 114, 99, 101, 32, 43, 61, 32, 34, 92, 114, 47, 47, 32, 34, 32, 43, 32, 101, 91, 105, 93, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 36, 40, 39, 35, 99, 111, 109, 112, 105, 108, 101, 100, 79, 117, 116, 112, 117, 116, 39, 41, 46, 118, 97, 108, 40, 99, 111, 109, 112, 105, 108, 101, 100, 83, 111, 117, 114, 99, 101, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 32, 32, 32, 32, 32, 32, 32, 32, 47, 47, 32, 80, 111, 112, 117, 108, 97, 116, 101, 32, 116, 104, 101, 32, 115, 116, 114, 97, 100, 97, 83, 114, 99, 32, 112, 97, 110, 101, 32, 119, 105, 116, 104, 32, 111, 110, 101, 32, 111, 102, 32, 116, 104, 101, 32, 98, 117, 105, 108, 116, 32, 105, 110, 32, 115, 97, 109, 112, 108, 101, 115, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 101, 120, 97, 109, 112, 108, 101, 83, 101, 108, 101, 99, 116, 105, 111, 110, 67, 104, 97, 110, 103, 101, 100, 40, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 101, 120, 97, 109, 112, 108, 101, 115, 32, 61, 32, 100, 111, 99, 117, 109, 101, 110, 116, 46, 103, 101, 116, 69, 108, 101, 109, 101, 110, 116, 66, 121, 73, 100, 40, 39, 101, 120, 97, 109, 112, 108, 101, 115, 39, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 118, 97, 114, 32, 115, 101, 108, 101, 99, 116, 101, 100, 69, 120, 97, 109, 112, 108, 101, 32, 61, 32, 101, 120, 97, 109, 112, 108, 101, 115, 46, 111, 112, 116, 105, 111, 110, 115, 91, 101, 120, 97, 109, 112, 108, 101, 115, 46, 115, 101, 108, 101, 99, 116, 101, 100, 73, 110, 100, 101, 120, 93, 46, 118, 97, 108, 117, 101, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 105, 102, 32, 40, 115, 101, 108, 101, 99, 116, 101, 100, 69, 120, 97, 109, 112, 108, 101, 32, 33, 61, 32, 34, 34, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 36, 46, 103, 101, 116, 40, 39, 101, 120, 97, 109, 112, 108, 101, 115, 47, 39, 32, 43, 32, 115, 101, 108, 101, 99, 116, 101, 100, 69, 120, 97, 109, 112, 108, 101, 44, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 40, 115, 114, 99, 84, 101, 120, 116, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 36, 40, 39, 35, 115, 116, 114, 97, 100, 97, 83, 114, 99, 39, 41, 46, 118, 97, 108, 40, 115, 114, 99, 84, 101, 120, 116, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 115, 101, 116, 84, 105, 109, 101, 111, 117, 116, 40, 115, 114, 99, 85, 112, 100, 97, 116, 101, 100, 44, 49, 48, 48, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 32, 102, 117, 110, 99, 116, 105, 111, 110, 32, 40, 101, 114, 114, 41, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 110, 115, 111, 108, 101, 46, 108, 111, 103, 40, 101, 114, 114, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 41, 59, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 32, 32, 32, 32, 32, 32, 32, 32, 125, 10, 10, 32, 32, 32, 32, 60, 47, 115, 99, 114, 105, 112, 116, 62, 10, 60, 47, 104, 101, 97, 100, 62, 10, 60, 98, 111, 100, 121, 62, 10, 32, 32, 32, 32, 60, 104, 49, 62, 84, 121, 112, 101, 83, 99, 114, 105, 112, 116, 60, 47, 104, 49, 62, 10, 32, 32, 32, 32, 60, 98, 114, 32, 47, 62, 10, 32, 32, 32, 32, 60, 115, 101, 108, 101, 99, 116, 32, 105, 100, 61, 34, 101, 120, 97, 109, 112, 108, 101, 115, 34, 32, 111, 110, 99, 104, 97, 110, 103, 101, 61, 39, 101, 120, 97, 109, 112, 108, 101, 83, 101, 108, 101, 99, 116, 105, 111, 110, 67, 104, 97, 110, 103, 101, 100, 40, 41, 39, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 111, 112, 116, 105, 111, 110, 32, 118, 97, 108, 117, 101, 61, 34, 34, 62, 83, 101, 108, 101, 99, 116, 46, 46, 46, 60, 47, 111, 112, 116, 105, 111, 110, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 111, 112, 116, 105, 111, 110, 32, 118, 97, 108, 117, 101, 61, 34, 115, 109, 97, 108, 108, 46, 116, 115, 34, 62, 83, 109, 97, 108, 108, 60, 47, 111, 112, 116, 105, 111, 110, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 111, 112, 116, 105, 111, 110, 32, 118, 97, 108, 117, 101, 61, 34, 101, 109, 112, 108, 111, 121, 101, 101, 46, 116, 115, 34, 62, 69, 109, 112, 108, 111, 121, 101, 101, 115, 60, 47, 111, 112, 116, 105, 111, 110, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 111, 112, 116, 105, 111, 110, 32, 118, 97, 108, 117, 101, 61, 34, 99, 111, 110, 119, 97, 121, 46, 116, 115, 34, 62, 67, 111, 110, 119, 97, 121, 32, 71, 97, 109, 101, 32, 111, 102, 32, 76, 105, 102, 101, 60, 47, 111, 112, 116, 105, 111, 110, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 111, 112, 116, 105, 111, 110, 32, 118, 97, 108, 117, 101, 61, 34, 116, 121, 112, 101, 115, 99, 114, 105, 112, 116, 46, 116, 115, 34, 62, 84, 121, 112, 101, 83, 99, 114, 105, 112, 116, 32, 67, 111, 109, 112, 105, 108, 101, 114, 60, 47, 111, 112, 116, 105, 111, 110, 62, 10, 32, 32, 32, 32, 60, 47, 115, 101, 108, 101, 99, 116, 62, 10, 10, 32, 32, 32, 32, 60, 100, 105, 118, 62, 10, 32, 32, 32, 32, 32, 32, 32, 32, 60, 116, 101, 120, 116, 97, 114, 101, 97, 32, 105, 100, 61, 39, 115, 116, 114, 97, 100, 97, 83, 114, 99, 39, 32, 114, 111, 119, 115, 61, 39, 52, 48, 39, 32, 99, 111, 108, 115, 61, 39, 56, 48, 39, 32, 111, 110, 99, 104, 97, 110, 103, 101, 61, 39, 115, 114, 99, 85, 112, 100, 97, 116, 101, 100, 40, 41, 39, 32, 111, 110, 107, 101, 121, 117, 112, 61, 39, 115, 114, 99, 85, 112, 100, 97, 116, 101, 100, 40, 41, 39, 32, 115, 112, 101, 108, 108, 99, 104, 101, 99, 107, 61, 34, 102, 97, 108, 115, 101, 34, 62, 10, 47, 47, 84, 121, 112, 101, 32, 121, 111, 117, 114, 32, 84, 121, 112, 101, 83, 99, 114, 105, 112, 116, 32, 104, 101, 114, 101, 46, 46, 46, 10, 32, 32, 32, 32, 32, 32, 60, 47, 116, 101, 120, 116, 97, 114, 101, 97, 62, 10, 32, 32, 32, 32, 32, 32, 60, 116, 101, 120, 116, 97, 114, 101, 97, 32, 105, 100, 61, 39, 99, 111, 109, 112, 105, 108, 101, 100, 79, 117, 116, 112, 117, 116, 39, 32, 114, 111, 119, 115, 61, 39, 52, 48, 39, 32, 99, 111, 108, 115, 61, 39, 56, 48, 39, 32, 115, 112, 101, 108, 108, 99, 104, 101, 99, 107, 61, 34, 102, 97, 108, 115, 101, 34, 62, 10, 47, 47, 67, 111, 109, 112, 105, 108, 101, 100, 32, 99, 111, 100, 101, 32, 119, 105, 108, 108, 32, 115, 104, 111, 119, 32, 117, 112, 32, 104, 101, 114, 101, 46, 46, 46, 10, 32, 32, 32, 32, 32, 32, 60, 47, 116, 101, 120, 116, 97, 114, 101, 97, 62, 10, 32, 32, 32, 32, 32, 32, 60, 98, 114, 32, 47, 62, 10, 32, 32, 32, 32, 32, 32, 60, 98, 117, 116, 116, 111, 110, 32, 111, 110, 99, 108, 105, 99, 107, 61, 39, 101, 120, 101, 99, 117, 116, 101, 40, 41, 39, 47, 62, 82, 117, 110, 60, 47, 98, 117, 116, 116, 111, 110, 62, 32, 10, 32, 32, 32, 32, 32, 32, 60, 100, 105, 118, 32, 105, 100, 61, 39, 99, 111, 109, 112, 105, 108, 97, 116, 105, 111, 110, 39, 62, 80, 114, 101, 115, 115, 32, 39, 114, 117, 110, 39, 32, 116, 111, 32, 101, 120, 101, 99, 117, 116, 101, 32, 99, 111, 100, 101, 46, 46, 46, 60, 47, 100, 105, 118, 62, 10, 32, 32, 32, 32, 32, 32, 60, 100, 105, 118, 32, 105, 100, 61, 39, 114, 101, 115, 117, 108, 116, 115, 39, 62, 46, 46, 46, 119, 114, 105, 116, 101, 32, 121, 111, 117, 114, 32, 114, 101, 115, 117, 108, 116, 115, 32, 105, 110, 116, 111, 32, 35, 114, 101, 115, 117, 108, 116, 115, 46, 46, 46, 60, 47, 100, 105, 118, 62, 10, 32, 32, 32, 32, 60, 47, 100, 105, 118, 62, 10, 32, 32, 32, 32, 60, 100, 105, 118, 32, 105, 100, 61, 39, 98, 111, 100, 39, 32, 115, 116, 121, 108, 101, 61, 39, 100, 105, 115, 112, 108, 97, 121, 58, 110, 111, 110, 101, 39, 62, 60, 47, 100, 105, 118, 62, 10, 60, 47, 98, 111, 100, 121, 62, 10, 60, 47, 104, 116, 109, 108, 62, 10 ]); +// This file is determined to be Windows-1252 unless candidateDetectEncoding is set +fixtures['some.shiftjis.1.txt'] = Uint8Array.from([82, 177, 82, 241, 82, 201, 82, 191, 82, 205]); + const lorem = getLorem(); // needle encoded from 'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя' @@ -418,4 +421,3 @@ Donec vehicula mauris eget lacus mollis venenatis et sed nibh. Nam sodales ligul tail: VSBuffer.fromString(`Vivamus iaculis, lacus efficitur faucibus porta, dui nulla facilisis ligula, ut sodales odio nunc id sapien. Cras viverra auctor ipsum, dapibus mattis neque dictum sed. Sed convallis fermentum molestie. Nulla facilisi turpis duis.`) }; } - diff --git a/src/vs/workbench/services/textfile/test/common/textFileService.io.test.ts b/src/vs/workbench/services/textfile/test/common/textFileService.io.test.ts index 5442b00a9aa..47aa789ae2a 100644 --- a/src/vs/workbench/services/textfile/test/common/textFileService.io.test.ts +++ b/src/vs/workbench/services/textfile/test/common/textFileService.io.test.ts @@ -592,6 +592,21 @@ export default function createSuite(params: Params) { assert.strictEqual(result.encoding, 'windows1252'); }); + test('readStream - autoguessEncoding (candidateGuessEncodings)', async () => { + // This file is determined to be Windows-1252 unless candidateDetectEncoding is set. + const resource = URI.file(join(testDir, 'some.shiftjis.1.txt')); + + const result = await service.readStream(resource, { autoGuessEncoding: true, candidateGuessEncodings: ['utf-8', 'shiftjis', 'euc-jp'] }); + assert.strictEqual(result.encoding, 'shiftjis'); + }); + + test('readStream - autoguessEncoding (candidateGuessEncodings is Empty)', async () => { + const resource = URI.file(join(testDir, 'some_cp1252.txt')); + + const result = await service.readStream(resource, { autoGuessEncoding: true, candidateGuessEncodings: [] }); + assert.strictEqual(result.encoding, 'windows1252'); + }); + test('readStream - FILE_IS_BINARY', async () => { const resource = URI.file(join(testDir, 'binary.txt')); diff --git a/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts b/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts index 9d86f5f146b..6806b4e1c56 100644 --- a/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts +++ b/src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts @@ -208,6 +208,14 @@ suite('Encoding', () => { assert.strictEqual(mimes.encoding, 'windows1252'); }); + test('autoGuessEncoding (candidateGuessEncodings - ShiftJIS)', async function () { + // This file is determined to be windows1252 unless candidateDetectEncoding is set. + const file = FileAccess.asFileUri('vs/workbench/services/textfile/test/node/encoding/fixtures/some.shiftjis.1.txt').fsPath; + const buffer = await readExactlyByFile(file, 512 * 8); + const mimes = await encoding.detectEncodingFromBuffer(buffer, true, ['utf8', 'shiftjis', 'eucjp']); + assert.strictEqual(mimes.encoding, 'shiftjis'); + }); + async function readAndDecodeFromDisk(path: string, fileEncoding: string | null) { return new Promise((resolve, reject) => { fs.readFile(path, (err, data) => { @@ -246,7 +254,7 @@ suite('Encoding', () => { Buffer.from([65, 66, 67]), ]); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); assert.ok(detected); assert.ok(stream); @@ -262,7 +270,7 @@ suite('Encoding', () => { Buffer.from([65, 66, 67]), ]); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 64, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); assert.ok(detected); assert.ok(stream); @@ -275,7 +283,7 @@ suite('Encoding', () => { const source = newWriteableBufferStream(); source.end(); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 512, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); assert.ok(detected); assert.ok(stream); @@ -288,7 +296,7 @@ suite('Encoding', () => { const path = FileAccess.asFileUri('vs/workbench/services/textfile/test/node/encoding/fixtures/some_utf16be.css').fsPath; const source = streamToBufferReadableStream(fs.createReadStream(path)); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 64, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); assert.strictEqual(detected.encoding, 'utf16be'); assert.strictEqual(detected.seemsBinary, false); @@ -301,7 +309,7 @@ suite('Encoding', () => { test('toDecodeStream - empty file', async function () { const path = FileAccess.asFileUri('vs/workbench/services/textfile/test/node/encoding/fixtures/empty.txt').fsPath; const source = streamToBufferReadableStream(fs.createReadStream(path)); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); const expected = await readAndDecodeFromDisk(path, detected.encoding); const actual = await readAllAsString(stream); @@ -318,7 +326,7 @@ suite('Encoding', () => { } const source = newTestReadableStream(buffers); - const { stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); const expected = new TextDecoder().decode(incompleteEmojis); const actual = await readAllAsString(stream); @@ -330,7 +338,7 @@ suite('Encoding', () => { const path = FileAccess.asFileUri('vs/workbench/services/textfile/test/node/encoding/fixtures/some_gbk.txt').fsPath; const source = streamToBufferReadableStream(fs.createReadStream(path)); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async () => 'gbk' }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async () => 'gbk' }); assert.ok(detected); assert.ok(stream); @@ -342,7 +350,7 @@ suite('Encoding', () => { const path = FileAccess.asFileUri('vs/workbench/services/textfile/test/node/encoding/fixtures/issue_102202.txt').fsPath; const source = streamToBufferReadableStream(fs.createReadStream(path)); - const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async () => 'utf-8' }); + const { detected, stream } = await encoding.toDecodeStream(source, { acceptTextOnly: true, minBytesRequiredForDetection: 4, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async () => 'utf-8' }); assert.ok(detected); assert.ok(stream); @@ -365,7 +373,7 @@ suite('Encoding', () => { let error: Error | undefined = undefined; try { - await encoding.toDecodeStream(source(), { acceptTextOnly: true, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + await encoding.toDecodeStream(source(), { acceptTextOnly: true, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); } catch (e) { error = e; } @@ -375,7 +383,7 @@ suite('Encoding', () => { // acceptTextOnly: false - const { detected, stream } = await encoding.toDecodeStream(source(), { acceptTextOnly: false, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 }); + const { detected, stream } = await encoding.toDecodeStream(source(), { acceptTextOnly: false, guessEncoding: false, candidateGuessEncodings: [], overwriteEncoding: async detected => detected || encoding.UTF8 }); assert.ok(detected); assert.strictEqual(detected.seemsBinary, true); diff --git a/src/vs/workbench/services/textfile/test/node/encoding/fixtures/some.shiftjis.1.txt b/src/vs/workbench/services/textfile/test/node/encoding/fixtures/some.shiftjis.1.txt new file mode 100644 index 00000000000..67acab8795e --- /dev/null +++ b/src/vs/workbench/services/textfile/test/node/encoding/fixtures/some.shiftjis.1.txt @@ -0,0 +1,2 @@ +// This file is determined to be Windows-1252 unless candidateDetectEncoding is set. +ɂ \ No newline at end of file