Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-28 12:33:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Incrementally decode find result (fixes #11842)

Browse Source
This commit is contained in:
Christof Marti
2016-10-07 17:20:17 -07:00
parent 4315160dc0
commit 73ba662500
1 changed files with 103 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
src/vs/workbench/services/search/node/fileSearch.ts
View File

@@ -6,7 +6,7 @@
'use strict';
import * as childProcess from 'child_process';
import {StringDecoder} from 'string_decoder';
import {StringDecoder, NodeStringDecoder} from 'string_decoder';
import {toErrorMessage} from 'vs/base/common/errorMessage';
import fs = require('fs');
import paths = require('path');
@@ -129,15 +129,14 @@ export class FileWalker {
if (!this.maxFilesize) {
if (platform.isMacintosh) {
this.traversal = Traversal.MacFind;
traverse = this.macFindTraversal;
traverse = this.findTraversal;
// Disable 'dir' for now (#11181, #11179, #11183, #11182).
// TS (2.0.2) warns about unreachable code. Using comments.
} /* else if (false && platform.isWindows) {
} /* else if (platform.isWindows) {
this.traversal = Traversal.WindowsDir;
traverse = this.windowsDirTraversal;
} */ else if (platform.isLinux) {
this.traversal = Traversal.LinuxFind;
traverse = this.linuxFindTraversal;
traverse = this.findTraversal;
}
}
@@ -169,21 +168,38 @@ export class FileWalker {
});
}
private macFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
const isMac = platform.isMacintosh;
let done = (err?: Error) => {
done = () => {};
cb(err);
};
let leftover = '';
let first = true;
const tree = this.initDirectoryTree();
const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
if (err) {
done(err);
return;
}
// Mac: uses NFD unicode form on disk, but we want NFC
const relativeFiles = strings.normalizeNFC(stdout).split('\n./');
relativeFiles[0] = relativeFiles[0].trim().substr(2);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
const relativeFiles = normalized.split('\n./');
if (first && normalized.length >= 2) {
first = false;
relativeFiles[0] = relativeFiles[0].trim().substr(2);
}
if (last) {
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}
} else {
leftover = relativeFiles.pop();
}
if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
@@ -191,65 +207,41 @@ export class FileWalker {
return;
}
this.matchFiles(rootFolder, relativeFiles, onResult);
this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
done();
if (last) {
this.matchDirectoryTree(tree, rootFolder, onResult);
done();
}
});
}
protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
if (err) {
done(err);
return;
}
// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
// const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
// this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
// if (err) {
// done(err);
// return;
// }
const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}
// const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
// relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
// const n = relativeFiles.length;
// relativeFiles[n - 1] = relativeFiles[n - 1].trim();
// if (!relativeFiles[n - 1]) {
// relativeFiles.pop();
// }
if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
done(new Error('Splitting up files failed'));
return;
}
// if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
// done(new Error('Splitting up files failed'));
// return;
// }
this.matchFiles(rootFolder, relativeFiles, onResult);
// this.matchFiles(rootFolder, relativeFiles, onResult);
done();
});
}
private linuxFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
if (err) {
done(err);
return;
}
const relativeFiles = stdout.split('\n./');
relativeFiles[0] = relativeFiles[0].trim().substr(2);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}
if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
done(new Error('Splitting up files failed'));
return;
}
this.matchFiles(rootFolder, relativeFiles, onResult);
done();
});
}
// done();
// });
// }
/**
* Public for testing.
@@ -279,13 +271,30 @@ export class FileWalker {
* Public for testing.
*/
public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
let done = (err: Error, stdout?: string) => {
done = () => {};
this.cmdForkResultTime = Date.now();
cb(err, stdout);
let all = '';
this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
if (err) {
cb(err);
return;
}
all += stdout;
if (last) {
cb(null, all);
}
});
}
private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
let done = (err: Error, stdout?: string, last?: boolean) => {
if (err || last) {
done = () => {};
this.cmdForkResultTime = Date.now();
}
cb(err, stdout, last);
};
const stdout = this.collectData(cmd.stdout);
this.forwardData(cmd.stdout, encoding, done);
const stderr = this.collectData(cmd.stderr);
cmd.on('error', (err: Error) => {
@@ -296,11 +305,19 @@ export class FileWalker {
if (code !== 0) {
done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
} else {
done(null, this.decodeData(stdout, encoding));
done(null, '', true);
}
});
}
private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
const decoder = new StringDecoder(encoding);
stream.on('data', (data: Buffer) => {
cb(null, decoder.write(data));
});
return decoder;
}
private collectData(stream: Readable): Buffer[] {
const buffers: Buffer[] = [];
stream.on('data', (data: Buffer) => {
@@ -314,27 +331,25 @@ export class FileWalker {
return buffers.map(buffer => decoder.write(buffer)).join('');
}
private matchFiles(rootFolder: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
this.cmdResultCount = relativeFiles.length;
// Support relative paths to files from a root resource (ignores excludes)
if (relativeFiles.indexOf(this.filePattern) !== -1) {
const basename = paths.basename(this.filePattern);
this.matchFile(onResult, { base: rootFolder, relativePath: this.filePattern, basename });
}
const tree = this.buildDirectoryTree(rootFolder, relativeFiles);
this.matchDirectoryTree(rootFolder, tree, onResult);
}
private buildDirectoryTree(base: string, relativeFilePaths: string[]): IDirectoryTree {
private initDirectoryTree(): IDirectoryTree {
const tree: IDirectoryTree = {
rootEntries: [],
pathToEntries: Object.create(null)
};
const {pathToEntries} = tree;
pathToEntries['.'] = tree.rootEntries;
relativeFilePaths.forEach(function add(relativePath: string) {
tree.pathToEntries['.'] = tree.rootEntries;
return tree;
}
private addDirectoryEntries({pathToEntries}: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
this.cmdResultCount += relativeFiles.length;
// Support relative paths to files from a root resource (ignores excludes)
if (relativeFiles.indexOf(this.filePattern) !== -1) {
const basename = paths.basename(this.filePattern);
this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
}
relativeFiles.forEach(function add(relativePath: string) {
const basename = paths.basename(relativePath);
const dirname = paths.dirname(relativePath);
let entries = pathToEntries[dirname];
@@ -348,10 +363,9 @@ export class FileWalker {
basename
});
});
return tree;
}
private matchDirectoryTree(rootFolder: string, { rootEntries, pathToEntries }: IDirectoryTree, onResult: (result: IRawFileMatch) => void) {
private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
const self = this;
const excludePattern = this.excludePattern;
const filePattern = this.filePattern;