Merge branch 'roblou/multirootsearch'

src/vs/workbench/services/search/node/fileSearch.ts

@@ -13,6 +13,7 @@ import paths = require('path');
 import { Readable } from 'stream';
 
 import scorer = require('vs/base/common/scorer');
+import objects = require('vs/base/common/objects');
 import arrays = require('vs/base/common/arrays');
 import platform = require('vs/base/common/platform');
 import strings = require('vs/base/common/strings');
@@ -22,7 +23,7 @@ import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/searc
 
 import extfs = require('vs/base/node/extfs');
 import flow = require('vs/base/node/flow');
-import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search';
+import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
 
 enum Traversal {
 	Node = 1,
@@ -46,7 +47,6 @@ export class FileWalker {
 	private config: IRawSearch;
 	private filePattern: string;
 	private normalizedFilePatternLowercase: string;
-	private excludePattern: glob.ParsedExpression;
 	private includePattern: glob.ParsedExpression;
 	private maxResults: number;
 	private maxFilesize: number;
@@ -62,12 +62,14 @@ export class FileWalker {
 	private cmdForkResultTime: number;
 	private cmdResultCount: number;
 
+	private folderExcludePatterns: Map<string, glob.ParsedExpression>;
+	private globalExcludePattern: glob.ParsedExpression;
+
 	private walkedPaths: { [path: string]: boolean; };
 
 	constructor(config: IRawSearch) {
 		this.config = config;
 		this.filePattern = config.filePattern;
-		this.excludePattern = glob.parse(config.excludePattern, { trimForExclusions: true });
 		this.includePattern = config.includePattern && glob.parse(config.includePattern);
 		this.maxResults = config.maxResults || null;
 		this.maxFilesize = config.maxFilesize || null;
@@ -82,13 +84,30 @@ export class FileWalker {
 		if (this.filePattern) {
 			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
 		}
+
+		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
+		this.folderExcludePatterns = new Map<string, glob.ParsedExpression>();
+
+		config.folderQueries.forEach(folderQuery => {
+			const folderExcludeExpression: glob.IExpression = objects.assign({}, this.config.excludePattern || {}, folderQuery.excludePattern || {});
+
+			// Add excludes for other root folders
+			config.folderQueries
+				.map(rootFolderQuery => rootFolderQuery.folder)
+				.filter(rootFolder => rootFolder !== folderQuery.folder)
+				.forEach(rootFolder => {
+					folderExcludeExpression[paths.join(rootFolder, '**/*')] = true;
+				});
+
+			this.folderExcludePatterns.set(folderQuery.folder, glob.parse(folderExcludeExpression));
+		});
 	}
 
 	public cancel(): void {
 		this.isCanceled = true;
 	}
 
-	public walk(rootFolders: string[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
+	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
 		this.fileWalkStartTime = Date.now();
 
 		// Support that the file pattern is a full path to a file that exists
@@ -116,7 +135,7 @@ export class FileWalker {
 			if (extraFiles) {
 				extraFiles.forEach(extraFilePath => {
 					const basename = paths.basename(extraFilePath);
-					if (this.excludePattern(extraFilePath, basename)) {
+					if (!this.globalExcludePattern || this.globalExcludePattern(extraFilePath, basename)) {
 						return; // excluded
 					}
 
@@ -146,8 +165,8 @@ export class FileWalker {
 			}
 
 			// For each root folder
-			flow.parallel<string, void>(rootFolders, (rootFolder: string, rootFolderDone: (err: Error, result: void) => void) => {
-				this.call(traverse, this, rootFolder, onResult, (err?: Error) => {
+			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
+				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
 					if (err) {
 						if (isNodeTraversal) {
 							rootFolderDone(err, undefined);
@@ -156,7 +175,7 @@ export class FileWalker {
 							const errorMessage = toErrorMessage(err);
 							console.error(errorMessage);
 							this.errors.push(errorMessage);
-							this.nodeJSTraversal(rootFolder, onResult, err => rootFolderDone(err, undefined));
+							this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined));
 						}
 					} else {
 						rootFolderDone(undefined, undefined);
@@ -176,7 +195,8 @@ export class FileWalker {
 		}
 	}
 
-	private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
+	private findTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
+		const rootFolder = folderQuery.folder;
 		const isMac = platform.isMacintosh;
 		let done = (err?: Error) => {
 			done = () => { };
@@ -185,7 +205,7 @@ export class FileWalker {
 		let leftover = '';
 		let first = true;
 		const tree = this.initDirectoryTree();
-		const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
+		const cmd = this.spawnFindCmd(folderQuery);
 		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
 			if (err) {
 				done(err);
@@ -254,17 +274,19 @@ export class FileWalker {
 	/**
 	 * Public for testing.
 	 */
-	public spawnFindCmd(rootFolder: string, excludePattern: glob.ParsedExpression) {
+	public spawnFindCmd(folderQuery: IFolderSearch) {
+		// Does this actually work for absolute paths for other roots?
+		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder);
 		const basenames = glob.getBasenameTerms(excludePattern);
-		const paths = glob.getPathTerms(excludePattern);
+		const pathTerms = glob.getPathTerms(excludePattern);
 		let args = ['-L', '.'];
-		if (basenames.length || paths.length) {
+		if (basenames.length || pathTerms.length) {
 			args.push('-not', '(', '(');
 			for (const basename of basenames) {
 				args.push('-name', basename);
 				args.push('-o');
 			}
-			for (const path of paths) {
+			for (const path of pathTerms) {
 				args.push('-path', path);
 				args.push('-o');
 			}
@@ -272,7 +294,7 @@ export class FileWalker {
 			args.push(')', '-prune', ')');
 		}
 		args.push('-type', 'f');
-		return childProcess.spawn('find', args, { cwd: rootFolder });
+		return childProcess.spawn('find', args, { cwd: folderQuery.folder });
 	}
 
 	/**
@@ -376,7 +398,7 @@ export class FileWalker {
 
 	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
 		const self = this;
-		const excludePattern = this.excludePattern;
+		const excludePattern = this.folderExcludePatterns.get(rootFolder);
 		const filePattern = this.filePattern;
 		function matchDirectory(entries: IDirectoryEntry[]) {
 			self.directoriesWalked++;
@@ -408,15 +430,15 @@ export class FileWalker {
 		matchDirectory(rootEntries);
 	}
 
-	private nodeJSTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
+	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
 		this.directoriesWalked++;
-		extfs.readdir(rootFolder, (error: Error, files: string[]) => {
+		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
 			if (error || this.isCanceled || this.isLimitHit) {
 				return done();
 			}
 
 			// Support relative paths to files from a root resource (ignores excludes)
-			return this.checkFilePatternRelativeMatch(rootFolder, (match, size) => {
+			return this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
 				if (this.isCanceled || this.isLimitHit) {
 					return done();
 				}
@@ -425,14 +447,14 @@ export class FileWalker {
 				if (match) {
 					this.resultCount++;
 					onResult({
-						base: rootFolder,
+						base: folderQuery.folder,
 						relativePath: this.filePattern,
 						basename: paths.basename(this.filePattern),
 						size
 					});
 				}
 
-				return this.doWalk(rootFolder, '', files, onResult, done);
+				return this.doWalk(folderQuery, '', files, onResult, done);
 			});
 		});
 	}
@@ -475,7 +497,8 @@ export class FileWalker {
 		});
 	}
 
-	private doWalk(rootFolder: string, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
+	private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
+		const rootFolder = folderQuery.folder;
 
 		// Execute tasks on each file in parallel to optimize throughput
 		flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => {
@@ -495,7 +518,7 @@ export class FileWalker {
 
 			// Check exclude pattern
 			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(paths.sep) : file;
-			if (this.excludePattern(currentRelativePath, file, () => siblings)) {
+			if (this.folderExcludePatterns.get(folderQuery.folder)(currentRelativePath, file, () => siblings)) {
 				return clb(null, undefined);
 			}
 
@@ -536,7 +559,7 @@ export class FileWalker {
 									return clb(null, undefined);
 								}
 
-								this.doWalk(rootFolder, currentRelativePath, children, onResult, err => clb(err, undefined));
+								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined));
 							});
 						});
 					}
@@ -621,19 +644,19 @@ export class FileWalker {
 }
 
 export class Engine implements ISearchEngine<IRawFileMatch> {
-	private rootFolders: string[];
+	private folderQueries: IFolderSearch[];
 	private extraFiles: string[];
 	private walker: FileWalker;
 
 	constructor(config: IRawSearch) {
-		this.rootFolders = config.folderQueries.map(folderQ => folderQ.folder);
+		this.folderQueries = config.folderQueries;
 		this.extraFiles = config.extraFiles;
 
 		this.walker = new FileWalker(config);
 	}
 
 	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
-		this.walker.walk(this.rootFolders, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
+		this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
 			done(err, {
 				limitHit: isLimitHit,
 				stats: this.walker.getStats()

src/vs/workbench/services/search/node/textSearch.ts

@@ -126,7 +126,7 @@ export class Engine implements ISearchEngine<ISerializedFileMatch[]> {
 		let nextBatch: string[] = [];
 		let nextBatchBytes = 0;
 		const batchFlushBytes = 2 ** 20; // 1MB
-		this.walker.walk(this.config.folderQueries.map(folderQ => folderQ.folder), this.config.extraFiles, result => {
+		this.walker.walk(this.config.folderQueries, this.config.extraFiles, result => {
 			let bytes = result.size || 1;
 			this.totalBytes += bytes;
 

src/vs/workbench/services/search/test/node/search.test.ts

@@ -8,17 +8,17 @@
 import path = require('path');
 import assert = require('assert');
 
-import * as glob from 'vs/base/common/glob';
 import { join, normalize } from 'vs/base/common/paths';
 import * as platform from 'vs/base/common/platform';
 
 import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch';
 import { IRawFileMatch, IFolderSearch } from 'vs/workbench/services/search/node/search';
 
-const TEST_ROOT_FOLDER = path.normalize(require.toUrl('./fixtures'));
+const TEST_FIXTURES = path.normalize(require.toUrl('./fixtures'));
+const TEST_ROOT_FOLDER: IFolderSearch = { folder: TEST_FIXTURES };
 function rootFolderQueries(): IFolderSearch[] {
 	return [
-		{ folder: TEST_ROOT_FOLDER }
+		TEST_ROOT_FOLDER
 	];
 }
 
@@ -440,17 +440,18 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const file0 = './more/file.txt';
 		const file1 = './examples/subfolder/subfile.txt';
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/something': true }));
+		const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/something': true } });
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1);
 
-			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder': true }));
+			const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder': true } });
+			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 			walker.readStdout(cmd2, 'utf8', (err2, stdout2) => {
 				assert.equal(err2, null);
 				assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
@@ -466,19 +467,20 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const file0 = './index.html';
 		const file1 = './examples/small.js';
 		const file2 = './more/file.txt';
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/something': true }));
+		const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/something': true } });
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file2), -1, stdout1);
 
-			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '{**/examples,**/more}': true }));
+			const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '{**/examples,**/more}': true } });
+			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 			walker.readStdout(cmd2, 'utf8', (err2, stdout2) => {
 				assert.equal(err2, null);
 				assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
@@ -495,17 +497,18 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const file0 = './examples/company.js';
 		const file1 = './examples/subfolder/subfile.txt';
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/examples/something': true }));
+		const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/examples/something': true } });
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1);
 
-			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/examples/subfolder': true }));
+			const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/examples/subfolder': true } });
+			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 			walker.readStdout(cmd2, 'utf8', (err2, stdout2) => {
 				assert.equal(err2, null);
 				assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
@@ -521,17 +524,18 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const file0 = './examples/subfolder/subfile.txt';
 		const file1 = './examples/subfolder/anotherfolder/anotherfile.txt';
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder/something': true }));
+		const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder/something': true } });
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1);
 
-			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ '**/subfolder/anotherfolder': true }));
+			const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { '**/subfolder/anotherfolder': true } });
+			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 			walker.readStdout(cmd2, 'utf8', (err2, stdout2) => {
 				assert.equal(err2, null);
 				assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
@@ -547,17 +551,18 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const file0 = './examples/company.js';
 		const file1 = './examples/subfolder/subfile.txt';
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ 'examples/something': true }));
+		const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { 'examples/something': true } });
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
 			assert.notStrictEqual(stdout1.split('\n').indexOf(file1), -1, stdout1);
 
-			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({ 'examples/subfolder': true }));
+			const walker = new FileWalker({ folderQueries: rootFolderQueries(), excludePattern: { 'examples/subfolder': true } });
+			const cmd2 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 			walker.readStdout(cmd2, 'utf8', (err2, stdout2) => {
 				assert.equal(err2, null);
 				assert.notStrictEqual(stdout1.split('\n').indexOf(file0), -1, stdout1);
@@ -573,7 +578,6 @@ suite('Search', () => {
 			return;
 		}
 
-		const walker = new FileWalker({ folderQueries: rootFolderQueries() });
 		const filesIn = [
 			'./examples/subfolder/subfile.txt',
 			'./examples/company.js',
@@ -584,12 +588,16 @@ suite('Search', () => {
 			'./more/file.txt'
 		];
 
-		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER, glob.parse({
-			'**/subfolder/anotherfolder': true,
-			'**/something/else': true,
-			'**/more': true,
-			'**/andmore': true
-		}));
+		const walker = new FileWalker({
+			folderQueries: rootFolderQueries(),
+			excludePattern: {
+				'**/subfolder/anotherfolder': true,
+				'**/something/else': true,
+				'**/more': true,
+				'**/andmore': true
+			}
+		});
+		const cmd1 = walker.spawnFindCmd(TEST_ROOT_FOLDER);
 		walker.readStdout(cmd1, 'utf8', (err1, stdout1) => {
 			assert.equal(err1, null);
 			for (const fileIn of filesIn) {