Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-23 10:08:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactor text seach

Browse Source 
To allow better code sharing between EH and search proc
This commit is contained in:
Rob Lourens
2018-10-15 11:43:37 -07:00
parent ee40fa3e0d
commit c26d6e6c93
15 changed files with 740 additions and 1139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

338
src/vs/workbench/api/node/extHostSearch.ts
View File

@@ -7,20 +7,22 @@ import * as path from 'path';
import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation';
import { toErrorMessage } from 'vs/base/common/errorMessage';
import * as glob from 'vs/base/common/glob';
import { toDisposable } from 'vs/base/common/lifecycle';
import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
import * as resources from 'vs/base/common/resources';
import { StopWatch } from 'vs/base/common/stopwatch';
import { URI, UriComponents } from 'vs/base/common/uri';
import { TPromise } from 'vs/base/common/winjs.base';
import * as extfs from 'vs/base/node/extfs';
import { ILogService } from 'vs/platform/log/common/log';
import { IFileMatch, IFileSearchProviderStats, IFolderQuery, IPatternInfo, IRawSearchQuery, ISearchCompleteStats, ISearchQuery, ITextSearchResult } from 'vs/platform/search/common/search';
import { IFileMatch, IFileSearchProviderStats, IFolderQuery, IPatternInfo, IRawSearchQuery, ISearchCompleteStats, ISearchQuery } from 'vs/platform/search/common/search';
import { ExtHostConfiguration } from 'vs/workbench/api/node/extHostConfiguration';
import { FileIndexSearchManager, IDirectoryEntry, IDirectoryTree, IInternalFileMatch, QueryGlobTester, resolvePatternsForProvider } from 'vs/workbench/api/node/extHostSearch.fileIndex';
import { FileIndexSearchManager, IDirectoryEntry, IDirectoryTree, IInternalFileMatch } from 'vs/workbench/api/node/extHostSearch.fileIndex';
import { RipgrepSearchProvider } from 'vs/workbench/services/search/node/ripgrepSearchEH';
import { OutputChannel } from 'vs/workbench/services/search/node/ripgrepSearchUtils';
import { QueryGlobTester, resolvePatternsForProvider } from 'vs/workbench/services/search/node/search';
import { TextSearchEngine } from 'vs/workbench/services/search/node/textSearchEngine';
import * as vscode from 'vscode';
import { ExtHostSearchShape, IMainContext, MainContext, MainThreadSearchShape } from './extHost.protocol';
import { RipgrepSearchProvider } from 'vs/workbench/services/search/node/ripgrepSearchEH';
export interface ISchemeTransformer {
transformOutgoing(scheme: string): string;
@@ -29,8 +31,9 @@ export interface ISchemeTransformer {
export class ExtHostSearch implements ExtHostSearchShape {
private readonly _proxy: MainThreadSearchShape;
private readonly _fileSearchProvider = new Map<number, vscode.FileSearchProvider>();
private readonly _textSearchProvider = new Map<number, vscode.TextSearchProvider>();
private readonly _fileSearchProvider = new Map<number, vscode.FileSearchProvider>();
private _internalFileSearchProvider;
private readonly _fileIndexProvider = new Map<number, vscode.FileIndexProvider>();
private _handlePool: number = 0;
@@ -52,17 +55,7 @@ export class ExtHostSearch implements ExtHostSearchShape {
return scheme;
}
registerFileSearchProvider(scheme: string, provider: vscode.FileSearchProvider) {
const handle = this._handlePool++;
this._fileSearchProvider.set(handle, provider);
this._proxy.$registerFileSearchProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._fileSearchProvider.delete(handle);
this._proxy.$unregisterProvider(handle);
});
}
registerTextSearchProvider(scheme: string, provider: vscode.TextSearchProvider) {
registerTextSearchProvider(scheme: string, provider: vscode.TextSearchProvider): IDisposable {
const handle = this._handlePool++;
this._textSearchProvider.set(handle, provider);
this._proxy.$registerTextSearchProvider(handle, this._transformScheme(scheme));
@@ -72,7 +65,27 @@ export class ExtHostSearch implements ExtHostSearchShape {
});
}
registerFileIndexProvider(scheme: string, provider: vscode.FileIndexProvider) {
registerFileSearchProvider(scheme: string, provider: vscode.FileSearchProvider): IDisposable {
const handle = this._handlePool++;
this._fileSearchProvider.set(handle, provider);
this._proxy.$registerFileSearchProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._fileSearchProvider.delete(handle);
this._proxy.$unregisterProvider(handle);
});
}
registerInternalFileSearchProvider(scheme: string, provider): IDisposable {
const handle = this._handlePool++;
this._internalFileSearchProvider = provider;
this._proxy.$registerFileSearchProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._internalFileSearchProvider = null;
this._proxy.$unregisterProvider(handle);
});
}
registerFileIndexProvider(scheme: string, provider: vscode.FileIndexProvider): IDisposable {
const handle = this._handlePool++;
this._fileIndexProvider.set(handle, provider);
this._proxy.$registerFileIndexProvider(handle, this._transformScheme(scheme));
@@ -98,6 +111,10 @@ export class ExtHostSearch implements ExtHostSearchShape {
}
$clearCache(cacheKey: string): Thenable<void> {
if (this._internalFileSearchProvider) {
this._internalFileSearchProvider.clearCache();
}
// Actually called once per provider.
// Only relevant to file index search.
return this._fileIndexSearchManager.clearCache(cacheKey);
@@ -117,11 +134,9 @@ export class ExtHostSearch implements ExtHostSearchShape {
function registerEHProviders(extHostSearch: ExtHostSearch, logService: ILogService, configService: ExtHostConfiguration) {
if (configService.getConfiguration('searchRipgrep').enable) {
console.log(`enabled`);
const outputChannel = new OutputChannel(logService);
extHostSearch.registerTextSearchProvider('file', new RipgrepSearchProvider(outputChannel));
}
const outputChannel = new OutputChannel(logService);
extHostSearch.registerTextSearchProvider('file', new RipgrepSearchProvider(outputChannel));
}
function reviveQuery(rawQuery: IRawSearchQuery): ISearchQuery {
@@ -141,287 +156,6 @@ function reviveFolderQuery(rawFolderQuery: IFolderQuery<UriComponents>): IFolder
};
}
class TextSearchResultsCollector {
private _batchedCollector: BatchedCollector<IFileMatch>;
private _currentFolderIdx: number;
private _currentUri: URI;
private _currentFileMatch: IFileMatch;
constructor(private _onResult: (result: IFileMatch[]) => void) {
this._batchedCollector = new BatchedCollector<IFileMatch>(512, items => this.sendItems(items));
}
add(data: vscode.TextSearchResult, folderIdx: number): void {
// Collects TextSearchResults into IInternalFileMatches and collates using BatchedCollector.
// This is efficient for ripgrep which sends results back one file at a time. It wouldn't be efficient for other search
// providers that send results in random order. We could do this step afterwards instead.
if (this._currentFileMatch && (this._currentFolderIdx !== folderIdx || !resources.isEqual(this._currentUri, data.uri))) {
this.pushToCollector();
this._currentFileMatch = null;
}
if (!this._currentFileMatch) {
this._currentFolderIdx = folderIdx;
this._currentFileMatch = {
resource: data.uri,
matches: []
};
}
this._currentFileMatch.matches.push(extensionResultToFrontendResult(data));
}
private pushToCollector(): void {
const size = this._currentFileMatch ?
this._currentFileMatch.matches.length :
0;
this._batchedCollector.addItem(this._currentFileMatch, size);
}
flush(): void {
this.pushToCollector();
this._batchedCollector.flush();
}
private sendItems(items: IFileMatch[]): void {
this._onResult(items);
}
}
function extensionResultToFrontendResult(data: vscode.TextSearchResult): ITextSearchResult {
// Warning: result from RipgrepTextSearchEH has fake vscode.Range. Don't depend on any other props beyond these...
return {
preview: {
match: {
startLineNumber: data.preview.match.start.line,
startColumn: data.preview.match.start.character,
endLineNumber: data.preview.match.end.line,
endColumn: data.preview.match.end.character
},
text: data.preview.text
},
range: {
startLineNumber: data.range.start.line,
startColumn: data.range.start.character,
endLineNumber: data.range.end.line,
endColumn: data.range.end.character
}
};
}
/**
* Collects items that have a size - before the cumulative size of collected items reaches START_BATCH_AFTER_COUNT, the callback is called for every
* set of items collected.
* But after that point, the callback is called with batches of maxBatchSize.
* If the batch isn't filled within some time, the callback is also called.
*/
class BatchedCollector<T> {
private static readonly TIMEOUT = 4000;
// After START_BATCH_AFTER_COUNT items have been collected, stop flushing on timeout
private static readonly START_BATCH_AFTER_COUNT = 50;
private totalNumberCompleted = 0;
private batch: T[] = [];
private batchSize = 0;
private timeoutHandle: any;
constructor(private maxBatchSize: number, private cb: (items: T[]) => void) {
}
addItem(item: T, size: number): void {
if (!item) {
return;
}
this.addItemToBatch(item, size);
}
addItems(items: T[], size: number): void {
if (!items) {
return;
}
if (this.maxBatchSize > 0) {
this.addItemsToBatch(items, size);
} else {
this.cb(items);
}
}
private addItemToBatch(item: T, size: number): void {
this.batch.push(item);
this.batchSize += size;
this.onUpdate();
}
private addItemsToBatch(item: T[], size: number): void {
this.batch = this.batch.concat(item);
this.batchSize += size;
this.onUpdate();
}
private onUpdate(): void {
if (this.totalNumberCompleted < BatchedCollector.START_BATCH_AFTER_COUNT) {
// Flush because we aren't batching yet
this.flush();
} else if (this.batchSize >= this.maxBatchSize) {
// Flush because the batch is full
this.flush();
} else if (!this.timeoutHandle) {
// No timeout running, start a timeout to flush
this.timeoutHandle = setTimeout(() => {
this.flush();
}, BatchedCollector.TIMEOUT);
}
}
flush(): void {
if (this.batchSize) {
this.totalNumberCompleted += this.batchSize;
this.cb(this.batch);
this.batch = [];
this.batchSize = 0;
if (this.timeoutHandle) {
clearTimeout(this.timeoutHandle);
this.timeoutHandle = 0;
}
}
}
}
class TextSearchEngine {
private collector: TextSearchResultsCollector;
private isLimitHit: boolean;
private resultCount = 0;
constructor(private pattern: IPatternInfo, private config: ISearchQuery, private provider: vscode.TextSearchProvider, private _extfs: typeof extfs) {
}
public search(onProgress: (matches: IFileMatch[]) => void, token: CancellationToken): TPromise<ISearchCompleteStats> {
const folderQueries = this.config.folderQueries;
const tokenSource = new CancellationTokenSource();
token.onCancellationRequested(() => tokenSource.cancel());
return new TPromise<ISearchCompleteStats>((resolve, reject) => {
this.collector = new TextSearchResultsCollector(onProgress);
let isCanceled = false;
const onResult = (match: vscode.TextSearchResult, folderIdx: number) => {
if (isCanceled) {
return;
}
if (this.resultCount >= this.config.maxResults) {
this.isLimitHit = true;
isCanceled = true;
tokenSource.cancel();
}
if (!this.isLimitHit) {
this.resultCount++;
this.collector.add(match, folderIdx);
}
};
// For each root folder
TPromise.join(folderQueries.map((fq, i) => {
return this.searchInFolder(fq, r => onResult(r, i), tokenSource.token);
})).then(results => {
tokenSource.dispose();
this.collector.flush();
const someFolderHitLImit = results.some(result => result && result.limitHit);
resolve({
limitHit: this.isLimitHit || someFolderHitLImit,
stats: {
type: 'textSearchProvider'
}
});
}, (errs: Error[]) => {
tokenSource.dispose();
const errMsg = errs
.map(err => toErrorMessage(err))
.filter(msg => !!msg)[0];
reject(new Error(errMsg));
});
});
}
private searchInFolder(folderQuery: IFolderQuery<URI>, onResult: (result: vscode.TextSearchResult) => void, token: CancellationToken): TPromise<vscode.TextSearchComplete> {
const queryTester = new QueryGlobTester(this.config, folderQuery);
const testingPs = [];
const progress = {
report: (result: vscode.TextSearchResult) => {
const hasSibling = folderQuery.folder.scheme === 'file' && glob.hasSiblingPromiseFn(() => {
return this.readdir(path.dirname(result.uri.fsPath));
});
const relativePath = path.relative(folderQuery.folder.fsPath, result.uri.fsPath);
testingPs.push(
queryTester.includedInQuery(relativePath, path.basename(relativePath), hasSibling)
.then(included => {
if (included) {
onResult(result);
}
}));
}
};
const searchOptions = this.getSearchOptionsForFolder(folderQuery);
return new TPromise(resolve => process.nextTick(resolve))
.then(() => this.provider.provideTextSearchResults(patternInfoToQuery(this.pattern), searchOptions, progress, token))
.then(result => {
return TPromise.join(testingPs)
.then(() => result);
});
}
private readdir(dirname: string): TPromise<string[]> {
return new TPromise((resolve, reject) => {
this._extfs.readdir(dirname, (err, files) => {
if (err) {
return reject(err);
}
resolve(files);
});
});
}
private getSearchOptionsForFolder(fq: IFolderQuery<URI>): vscode.TextSearchOptions {
const includes = resolvePatternsForProvider(this.config.includePattern, fq.includePattern);
const excludes = resolvePatternsForProvider(this.config.excludePattern, fq.excludePattern);
return {
folder: URI.from(fq.folder),
excludes,
includes,
useIgnoreFiles: !this.config.disregardIgnoreFiles,
useGlobalIgnoreFiles: !this.config.disregardGlobalIgnoreFiles,
followSymlinks: !this.config.ignoreSymlinks,
encoding: this.config.fileEncoding,
maxFileSize: this.config.maxFileSize,
maxResults: this.config.maxResults,
previewOptions: this.config.previewOptions
};
}
}
function patternInfoToQuery(patternInfo: IPatternInfo): vscode.TextSearchQuery {
return <vscode.TextSearchQuery>{
isCaseSensitive: patternInfo.isCaseSensitive || false,
isRegExp: patternInfo.isRegExp || false,
isWordMatch: patternInfo.isWordMatch || false,
pattern: patternInfo.pattern
};
}
class FileSearchEngine {
private filePattern: string;
private includePattern: glob.ParsedExpression;