From f686a20863de110239dcdc47a164fa576b2938ea Mon Sep 17 00:00:00 2001 From: Johannes Rieken Date: Tue, 10 Apr 2018 16:58:53 +0200 Subject: [PATCH] add provider to readable-stream logic, #41985 --- src/vs/base/node/encoding.ts | 34 ++++---- .../base/test/node/encoding/encoding.test.ts | 10 +-- .../electron-browser/remoteFileService.ts | 87 +++++++++++-------- 3 files changed, 72 insertions(+), 59 deletions(-) diff --git a/src/vs/base/node/encoding.ts b/src/vs/base/node/encoding.ts index 6318f2cbc5fd..7a35237f3e99 100644 --- a/src/vs/base/node/encoding.ts +++ b/src/vs/base/node/encoding.ts @@ -18,12 +18,21 @@ export const UTF16be = 'utf16be'; export const UTF16le = 'utf16le'; export interface IDecodeStreamOptions { - minBytesRequiredForDetection: number; - guessEncoding: boolean; - overwriteEncoding(detected: string): string; + guessEncoding?: boolean; + minBytesRequiredForDetection?: number; + overwriteEncoding?(detected: string): string; } -export function toDecodeStream(readable: Readable, opts: IDecodeStreamOptions): TPromise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }> { +export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): TPromise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }> { + + if (!options.minBytesRequiredForDetection) { + options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_GUESS_BUFFER_MAX_LEN : NO_GUESS_BUFFER_MAX_LEN; + } + + if (!options.overwriteEncoding) { + options.overwriteEncoding = detected => detected || UTF8; + } + return new TPromise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }>((resolve, reject) => { readable.pipe(new class extends Writable { @@ -55,7 +64,7 @@ export function toDecodeStream(readable: Readable, opts: IDecodeStreamOptions): // waiting for the decoder to be ready this._decodeStreamConstruction.then(_ => callback(), err => callback(err)); - } else if (this._bytesBuffered >= opts.minBytesRequiredForDetection) { + } else if (this._bytesBuffered >= options.minBytesRequiredForDetection) { // buffered enough data, create stream and forward data this._startDecodeStream(callback); @@ -69,8 +78,8 @@ export function toDecodeStream(readable: Readable, opts: IDecodeStreamOptions): this._decodeStreamConstruction = TPromise.as(detectEncodingFromBuffer({ buffer: Buffer.concat(this._buffer), bytesRead: this._bytesBuffered - }, opts.guessEncoding)).then(detected => { - detected.encoding = opts.overwriteEncoding(detected.encoding); // default encoding + }, options.guessEncoding)).then(detected => { + detected.encoding = options.overwriteEncoding(detected.encoding); // default encoding this._decodeStream = decodeStream(detected.encoding); for (const buffer of this._buffer) { this._decodeStream.write(buffer); @@ -256,17 +265,6 @@ const ZERO_BYTE_DETECTION_BUFFER_MAX_LEN = 512; // number of bytes to look at to const NO_GUESS_BUFFER_MAX_LEN = 512; // when not auto guessing the encoding, small number of bytes are enough const AUTO_GUESS_BUFFER_MAX_LEN = 512 * 8; // with auto guessing we want a lot more content to be read for guessing -export function maxEncodingDetectionBufferLen(arg1?: DetectEncodingOption | boolean): number { - let autoGuessEncoding: boolean; - if (typeof arg1 === 'boolean') { - autoGuessEncoding = arg1; - } else { - autoGuessEncoding = arg1 && arg1.autoGuessEncoding; - } - - return autoGuessEncoding ? AUTO_GUESS_BUFFER_MAX_LEN : NO_GUESS_BUFFER_MAX_LEN; -} - export interface IDetectedEncodingResult { encoding: string; seemsBinary: boolean; diff --git a/src/vs/base/test/node/encoding/encoding.test.ts b/src/vs/base/test/node/encoding/encoding.test.ts index 063912953f01..252ee281e377 100644 --- a/src/vs/base/test/node/encoding/encoding.test.ts +++ b/src/vs/base/test/node/encoding/encoding.test.ts @@ -189,7 +189,7 @@ suite('Encoding', () => { } }); - let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: true, overwriteEncoding() { return encoding.UTF8; } }); + let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4 }); assert.ok(detected); assert.ok(stream); @@ -209,7 +209,7 @@ suite('Encoding', () => { } }); - let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: true, overwriteEncoding() { return encoding.UTF8; } }); + let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64 }); assert.ok(detected); assert.ok(stream); @@ -226,7 +226,7 @@ suite('Encoding', () => { } }); - let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: true, overwriteEncoding() { return encoding.UTF8; } }); + let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512 }); assert.ok(detected); assert.ok(stream); @@ -241,7 +241,7 @@ suite('Encoding', () => { let path = require.toUrl('./fixtures/some_utf16be.css'); let source = fs.createReadStream(path); - let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: true, overwriteEncoding(detected) { return detected; } }); + let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64 }); assert.equal(detected.encoding, 'utf16be'); assert.equal(detected.seemsBinary, false); @@ -256,7 +256,7 @@ suite('Encoding', () => { let path = require.toUrl('./fixtures/empty.txt'); let source = fs.createReadStream(path); - let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: true, overwriteEncoding() { return encoding.UTF8; } }); + let { detected, stream } = await encoding.toDecodeStream(source, {}); let expected = await readAndDecodeFromDisk(path, detected.encoding); let actual = await readAllAsString(stream); diff --git a/src/vs/workbench/services/files/electron-browser/remoteFileService.ts b/src/vs/workbench/services/files/electron-browser/remoteFileService.ts index 84097d963311..8a4a0c41cc79 100644 --- a/src/vs/workbench/services/files/electron-browser/remoteFileService.ts +++ b/src/vs/workbench/services/files/electron-browser/remoteFileService.ts @@ -12,7 +12,7 @@ import { posix } from 'path'; import { IDisposable } from 'vs/base/common/lifecycle'; import { isFalsyOrEmpty, distinct } from 'vs/base/common/arrays'; import { Schemas } from 'vs/base/common/network'; -import { decodeStream, encode, UTF8, UTF8_with_bom, detectEncodingFromBuffer, maxEncodingDetectionBufferLen } from 'vs/base/node/encoding'; +import { encode, UTF8, UTF8_with_bom, toDecodeStream } from 'vs/base/node/encoding'; import { TernarySearchTree } from 'vs/base/common/map'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace'; @@ -23,6 +23,7 @@ import { ITextResourceConfigurationService } from 'vs/editor/common/services/res import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions'; import { localize } from 'vs/nls'; import { INotificationService } from 'vs/platform/notification/common/notification'; +import { Readable } from 'stream'; function toIFileStat(provider: IFileSystemProvider, tuple: [URI, IStat], recurse?: (tuple: [URI, IStat]) => boolean): TPromise { const [resource, stat] = tuple; @@ -220,6 +221,25 @@ export class RemoteFileService extends FileService { } } + private _createReadStream(provider: IFileSystemProvider, resource: URI): Readable { + return new class extends Readable { + _done: boolean = false; + _read(size?: number): void { + if (this._done) { + this.push(null); + return; + } + provider.readFile(resource).then(data => { + this._done = true; + this.push(data); + }, err => { + this._done = true; + this.emit('error', err); + }); + } + }; + } + private _readFile(resource: URI, options: IResolveContentOptions = Object.create(null)): TPromise { return this._withProvider(resource).then(provider => { @@ -242,15 +262,34 @@ export class RemoteFileService extends FileService { ); } - const guessEncoding = options.autoGuessEncoding; - const count = maxEncodingDetectionBufferLen(options); - let buffer: Buffer; + return toDecodeStream(this._createReadStream(provider, resource), { + guessEncoding: options.autoGuessEncoding, + overwriteEncoding: detected => { + let preferredEncoding: string; + if (options && options.encoding) { + if (detected === UTF8 && options.encoding === UTF8) { + preferredEncoding = UTF8_with_bom; // indicate the file has BOM if we are to resolve with UTF 8 + } else { + preferredEncoding = options.encoding; // give passed in encoding highest priority + } + } else if (detected) { + if (detected === UTF8) { + preferredEncoding = UTF8_with_bom; // if we detected UTF-8, it can only be because of a BOM + } else { + preferredEncoding = detected; + } + // todo@remote - encoding logic should not be kept + // hostage inside the node file service + // } else if (super.configuredEncoding(resource) === UTF8_with_bom) { + } else { + preferredEncoding = UTF8; // if we did not detect UTF 8 BOM before, this can only be UTF 8 then + } + return preferredEncoding; + } - return provider.readFile(resource).then(data => { - buffer = Buffer.from(data); - return detectEncodingFromBuffer({ bytesRead: Math.min(count, buffer.length), buffer }, guessEncoding); - }).then(detected => { - if (options.acceptTextOnly && detected.seemsBinary) { + }).then(data => { + + if (options.acceptTextOnly && data.detected.seemsBinary) { return TPromise.wrapError(new FileOperationError( localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), FileOperationResult.FILE_IS_BINARY, @@ -258,33 +297,9 @@ export class RemoteFileService extends FileService { )); } - let preferredEncoding: string; - if (options && options.encoding) { - if (detected.encoding === UTF8 && options.encoding === UTF8) { - preferredEncoding = UTF8_with_bom; // indicate the file has BOM if we are to resolve with UTF 8 - } else { - preferredEncoding = options.encoding; // give passed in encoding highest priority - } - } else if (detected.encoding) { - if (detected.encoding === UTF8) { - preferredEncoding = UTF8_with_bom; // if we detected UTF-8, it can only be because of a BOM - } else { - preferredEncoding = detected.encoding; - } - // todo@remote - encoding logic should not be kept - // hostage inside the node file service - // } else if (super.configuredEncoding(resource) === UTF8_with_bom) { - } else { - preferredEncoding = UTF8; // if we did not detect UTF 8 BOM before, this can only be UTF 8 then - } - - // const encoding = this.getEncoding(resource); - const stream = decodeStream(preferredEncoding); - stream.end(buffer); - - return { - encoding: preferredEncoding, - value: stream, + return { + encoding: data.detected.encoding, + value: data.stream, resource: fileStat.resource, name: fileStat.name, etag: fileStat.etag,