diff --git a/src/vs/workbench/services/search/node/fileSearch.ts b/src/vs/workbench/services/search/node/fileSearch.ts index 0b7a68ffc11..1d65b05af4f 100644 --- a/src/vs/workbench/services/search/node/fileSearch.ts +++ b/src/vs/workbench/services/search/node/fileSearch.ts @@ -13,6 +13,7 @@ import paths = require('path'); import { Readable } from 'stream'; import scorer = require('vs/base/common/scorer'); +import objects = require('vs/base/common/objects'); import arrays = require('vs/base/common/arrays'); import platform = require('vs/base/common/platform'); import strings = require('vs/base/common/strings'); @@ -22,7 +23,7 @@ import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/searc import extfs = require('vs/base/node/extfs'); import flow = require('vs/base/node/flow'); -import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search'; +import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search'; enum Traversal { Node = 1, @@ -46,7 +47,6 @@ export class FileWalker { private config: IRawSearch; private filePattern: string; private normalizedFilePatternLowercase: string; - private excludePattern: glob.ParsedExpression; private includePattern: glob.ParsedExpression; private maxResults: number; private maxFilesize: number; @@ -62,12 +62,14 @@ export class FileWalker { private cmdForkResultTime: number; private cmdResultCount: number; + private folderExcludePatterns: Map; + private globalExcludePattern: glob.ParsedExpression; + private walkedPaths: { [path: string]: boolean; }; constructor(config: IRawSearch) { this.config = config; this.filePattern = config.filePattern; - this.excludePattern = glob.parse(config.excludePattern, { trimForExclusions: true }); this.includePattern = config.includePattern && glob.parse(config.includePattern); this.maxResults = config.maxResults || null; this.maxFilesize = config.maxFilesize || null; @@ -82,13 +84,30 @@ export class FileWalker { if (this.filePattern) { this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase(); } + + this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern); + this.folderExcludePatterns = new Map(); + + config.folderQueries.forEach(folderQuery => { + const folderExcludeExpression: glob.IExpression = objects.assign({}, this.config.excludePattern || {}, folderQuery.excludePattern || {}); + + // Add excludes for other root folders + config.folderQueries + .map(rootFolderQuery => rootFolderQuery.folder) + .filter(rootFolder => rootFolder !== folderQuery.folder) + .forEach(rootFolder => { + folderExcludeExpression[paths.join(rootFolder, '**/*')] = true; + }); + + this.folderExcludePatterns.set(folderQuery.folder, glob.parse(folderExcludeExpression)); + }); } public cancel(): void { this.isCanceled = true; } - public walk(rootFolders: string[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void { + public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void { this.fileWalkStartTime = Date.now(); // Support that the file pattern is a full path to a file that exists @@ -116,7 +135,7 @@ export class FileWalker { if (extraFiles) { extraFiles.forEach(extraFilePath => { const basename = paths.basename(extraFilePath); - if (this.excludePattern(extraFilePath, basename)) { + if (!this.globalExcludePattern || this.globalExcludePattern(extraFilePath, basename)) { return; // excluded } @@ -146,8 +165,8 @@ export class FileWalker { } // For each root folder - flow.parallel(rootFolders, (rootFolder: string, rootFolderDone: (err: Error, result: void) => void) => { - this.call(traverse, this, rootFolder, onResult, (err?: Error) => { + flow.parallel(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => { + this.call(traverse, this, folderQuery, onResult, (err?: Error) => { if (err) { if (isNodeTraversal) { rootFolderDone(err, undefined); @@ -156,7 +175,7 @@ export class FileWalker { const errorMessage = toErrorMessage(err); console.error(errorMessage); this.errors.push(errorMessage); - this.nodeJSTraversal(rootFolder, onResult, err => rootFolderDone(err, undefined)); + this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined)); } } else { rootFolderDone(undefined, undefined); @@ -176,7 +195,8 @@ export class FileWalker { } } - private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void { + private findTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void { + const rootFolder = folderQuery.folder; const isMac = platform.isMacintosh; let done = (err?: Error) => { done = () => { }; @@ -185,7 +205,7 @@ export class FileWalker { let leftover = ''; let first = true; const tree = this.initDirectoryTree(); - const cmd = this.spawnFindCmd(rootFolder, this.excludePattern); + const cmd = this.spawnFindCmd(folderQuery); this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => { if (err) { done(err); @@ -254,17 +274,19 @@ export class FileWalker { /** * Public for testing. */ - public spawnFindCmd(rootFolder: string, excludePattern: glob.ParsedExpression) { + public spawnFindCmd(folderQuery: IFolderSearch) { + // Does this actually work for absolute paths for other roots? + const excludePattern = this.folderExcludePatterns.get(folderQuery.folder); const basenames = glob.getBasenameTerms(excludePattern); - const paths = glob.getPathTerms(excludePattern); + const pathTerms = glob.getPathTerms(excludePattern); let args = ['-L', '.']; - if (basenames.length || paths.length) { + if (basenames.length || pathTerms.length) { args.push('-not', '(', '('); for (const basename of basenames) { args.push('-name', basename); args.push('-o'); } - for (const path of paths) { + for (const path of pathTerms) { args.push('-path', path); args.push('-o'); } @@ -272,7 +294,7 @@ export class FileWalker { args.push(')', '-prune', ')'); } args.push('-type', 'f'); - return childProcess.spawn('find', args, { cwd: rootFolder }); + return childProcess.spawn('find', args, { cwd: folderQuery.folder }); } /** @@ -376,7 +398,7 @@ export class FileWalker { private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) { const self = this; - const excludePattern = this.excludePattern; + const excludePattern = this.folderExcludePatterns.get(rootFolder); const filePattern = this.filePattern; function matchDirectory(entries: IDirectoryEntry[]) { self.directoriesWalked++; @@ -408,15 +430,15 @@ export class FileWalker { matchDirectory(rootEntries); } - private nodeJSTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { + private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { this.directoriesWalked++; - extfs.readdir(rootFolder, (error: Error, files: string[]) => { + extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => { if (error || this.isCanceled || this.isLimitHit) { return done(); } // Support relative paths to files from a root resource (ignores excludes) - return this.checkFilePatternRelativeMatch(rootFolder, (match, size) => { + return this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => { if (this.isCanceled || this.isLimitHit) { return done(); } @@ -425,14 +447,14 @@ export class FileWalker { if (match) { this.resultCount++; onResult({ - base: rootFolder, + base: folderQuery.folder, relativePath: this.filePattern, basename: paths.basename(this.filePattern), size }); } - return this.doWalk(rootFolder, '', files, onResult, done); + return this.doWalk(folderQuery, '', files, onResult, done); }); }); } @@ -475,7 +497,8 @@ export class FileWalker { }); } - private doWalk(rootFolder: string, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void { + private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void { + const rootFolder = folderQuery.folder; // Execute tasks on each file in parallel to optimize throughput flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => { @@ -495,7 +518,7 @@ export class FileWalker { // Check exclude pattern let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(paths.sep) : file; - if (this.excludePattern(currentRelativePath, file, () => siblings)) { + if (this.folderExcludePatterns.get(folderQuery.folder)(currentRelativePath, file, () => siblings)) { return clb(null, undefined); } @@ -536,7 +559,7 @@ export class FileWalker { return clb(null, undefined); } - this.doWalk(rootFolder, currentRelativePath, children, onResult, err => clb(err, undefined)); + this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined)); }); }); } @@ -621,19 +644,19 @@ export class FileWalker { } export class Engine implements ISearchEngine { - private rootFolders: string[]; + private folderQueries: IFolderSearch[]; private extraFiles: string[]; private walker: FileWalker; constructor(config: IRawSearch) { - this.rootFolders = config.folderQueries.map(folderQ => folderQ.folder); + this.folderQueries = config.folderQueries; this.extraFiles = config.extraFiles; this.walker = new FileWalker(config); } public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { - this.walker.walk(this.rootFolders, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => { + this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => { done(err, { limitHit: isLimitHit, stats: this.walker.getStats() diff --git a/src/vs/workbench/services/search/node/textSearch.ts b/src/vs/workbench/services/search/node/textSearch.ts index ef0954a9cbe..341b100b5aa 100644 --- a/src/vs/workbench/services/search/node/textSearch.ts +++ b/src/vs/workbench/services/search/node/textSearch.ts @@ -126,7 +126,7 @@ export class Engine implements ISearchEngine { let nextBatch: string[] = []; let nextBatchBytes = 0; const batchFlushBytes = 2 ** 20; // 1MB - this.walker.walk(this.config.folderQueries.map(folderQ => folderQ.folder), this.config.extraFiles, result => { + this.walker.walk(this.config.folderQueries, this.config.extraFiles, result => { let bytes = result.size || 1; this.totalBytes += bytes; diff --git a/src/vs/workbench/services/search/test/node/search.test.ts b/src/vs/workbench/services/search/test/node/search.test.ts index e8a8c265b07..308f482f8b6 100644 --- a/src/vs/workbench/services/search/test/node/search.test.ts +++ b/src/vs/workbench/services/search/test/node/search.test.ts @@ -8,17 +8,17 @@ import path = require('path'); import assert = require('assert'); -import * as glob from 'vs/base/common/glob'; import { join, normalize } from 'vs/base/common/paths'; import * as platform from 'vs/base/common/platform'; import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch'; import { IRawFileMatch, IFolderSearch } from 'vs/workbench/services/search/node/search'; -const TEST_ROOT_FOLDER = path.normalize(require.toUrl('./fixtures')); +const TEST_FIXTURES = path.normalize(require.toUrl('./fixtures')); +const TEST_ROOT_FOLDER: IFolderSearch = { folder: TEST_FIXTURES }; function rootFolderQueries(): IFolderSearch[] { return [ - { folder: TEST_ROOT_FOLDER } + TEST_ROOT_FOLDER ]; } @@ -440,17 +440,18 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const file0 = './more/file.txt'; const file1 = './examples/subfolder/subfile.txt'; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/something': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/something': true } }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1); - const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder': true } }); + const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd2, 'utf8', (err2, stdout2) => { assert.equal(err2, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); @@ -466,19 +467,20 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const file0 = './index.html'; const file1 = './examples/small.js'; const file2 = './more/file.txt'; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/something': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/something': true } }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file2), -1, stdout1); - const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '{**/examples,**/more}': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '{**/examples,**/more}': true } }); + const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd2, 'utf8', (err2, stdout2) => { assert.equal(err2, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); @@ -495,17 +497,18 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const file0 = './examples/company.js'; const file1 = './examples/subfolder/subfile.txt'; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/examples/something': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/examples/something': true } }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1); - const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/examples/subfolder': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/examples/subfolder': true } }); + const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd2, 'utf8', (err2, stdout2) => { assert.equal(err2, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); @@ -521,17 +524,18 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const file0 = './examples/subfolder/subfile.txt'; const file1 = './examples/subfolder/anotherfolder/anotherfile.txt'; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder/something': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder/something': true } }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1); - const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder/anotherfolder': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder/anotherfolder': true } }); + const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd2, 'utf8', (err2, stdout2) => { assert.equal(err2, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); @@ -547,17 +551,18 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const file0 = './examples/company.js'; const file1 = './examples/subfolder/subfile.txt'; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ 'examples/something': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { 'examples/something': true } }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1); - const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ 'examples/subfolder': true })); + const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { 'examples/subfolder': true } }); + const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd2, 'utf8', (err2, stdout2) => { assert.equal(err2, null); assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1); @@ -573,7 +578,6 @@ suite('Search', () => { return; } - const walker = new FileWalker({ folderQueries: rootFolderQueries() }); const filesIn = [ './examples/subfolder/subfile.txt', './examples/company.js', @@ -584,12 +588,16 @@ suite('Search', () => { './more/file.txt' ]; - const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ - '**/subfolder/anotherfolder': true, - '**/something/else': true, - '**/more': true, - '**/andmore': true - })); + const walker = new FileWalker({ + folderQueries: rootFolderQueries(), + excludePattern: { + '**/subfolder/anotherfolder': true, + '**/something/else': true, + '**/more': true, + '**/andmore': true + } + }); + const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER); walker.readStdout(cmd1, 'utf8', (err1, stdout1) => { assert.equal(err1, null); for (const fileIn of filesIn) {