From a84db4acdea2c96038c45833e07d8df4781845aa Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Wed, 9 May 2018 17:09:05 -0700 Subject: [PATCH] EH search - quickopen basically working (without filtering, batched results, etc) --- extensions/search-rg/src/extension.ts | 6 + extensions/search-rg/src/normalization.ts | 36 ++++ extensions/search-rg/src/ripgrepFileSearch.ts | 202 ++++++++++++++++++ extensions/search-rg/src/ripgrepHelpers.ts | 11 - extensions/search-rg/src/ripgrepTextSearch.ts | 9 +- src/vs/vscode.proposed.d.ts | 4 +- .../api/electron-browser/mainThreadSearch.ts | 2 +- src/vs/workbench/api/node/extHost.protocol.ts | 2 +- src/vs/workbench/api/node/extHostSearch.ts | 54 +++-- 9 files changed, 295 insertions(+), 31 deletions(-) create mode 100644 extensions/search-rg/src/normalization.ts create mode 100644 extensions/search-rg/src/ripgrepFileSearch.ts diff --git a/extensions/search-rg/src/extension.ts b/extensions/search-rg/src/extension.ts index 160bba46ff7..4ef12c454d1 100644 --- a/extensions/search-rg/src/extension.ts +++ b/extensions/search-rg/src/extension.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode'; import { RipgrepTextSearchEngine } from './ripgrepTextSearch'; +import { RipgrepFileSearchEngine } from './ripgrepFileSearch'; export function activate(): void { const provider = new RipgrepSearchProvider(); @@ -16,4 +17,9 @@ class RipgrepSearchProvider implements vscode.SearchProvider { const engine = new RipgrepTextSearchEngine(); return engine.provideTextSearchResults(query, options, progress, token); } + + provideFileSearchResults(options: vscode.SearchOptions, progress: vscode.Progress, token: vscode.CancellationToken): Thenable { + const engine = new RipgrepFileSearchEngine(); + return engine.provideFileSearchResults(options, progress, token); + } } \ No newline at end of file diff --git a/extensions/search-rg/src/normalization.ts b/extensions/search-rg/src/normalization.ts new file mode 100644 index 00000000000..e795688c5a1 --- /dev/null +++ b/extensions/search-rg/src/normalization.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * The normalize() method returns the Unicode Normalization Form of a given string. The form will be + * the Normalization Form Canonical Composition. + * + * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize} + */ +export const canNormalize = typeof (('').normalize) === 'function'; + +export function normalizeNFC(str: string): string { + return normalize(str, 'NFC'); +} + +export function normalizeNFD(str: string): string { + return normalize(str, 'NFD'); +} + +const nonAsciiCharactersPattern = /[^\u0000-\u0080]/; +function normalize(str: string, form: string): string { + if (!canNormalize || !str) { + return str; + } + + let res: string; + if (nonAsciiCharactersPattern.test(str)) { + res = (str).normalize(form); + } else { + res = str; + } + + return res; +} diff --git a/extensions/search-rg/src/ripgrepFileSearch.ts b/extensions/search-rg/src/ripgrepFileSearch.ts new file mode 100644 index 00000000000..e5b4474689c --- /dev/null +++ b/extensions/search-rg/src/ripgrepFileSearch.ts @@ -0,0 +1,202 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as cp from 'child_process'; +import * as path from 'path'; +import { Readable } from 'stream'; +import { NodeStringDecoder, StringDecoder } from 'string_decoder'; +import * as vscode from 'vscode'; +import { rgPath } from 'vscode-ripgrep'; +import { normalizeNFC, normalizeNFD } from './normalization'; +import { rgErrorMsgForDisplay } from './ripgrepTextSearch'; + +const isMac = process.platform === 'darwin'; + +// If vscode-ripgrep is in an .asar file, then the binary is unpacked. +const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked'); + +export class RipgrepFileSearchEngine { + private isDone = false; + private rgProc: cp.ChildProcess; + private killRgProcFn: (code?: number) => void; + + constructor() { + this.killRgProcFn = () => this.rgProc && this.rgProc.kill(); + } + + provideFileSearchResults(options: vscode.SearchOptions, progress: vscode.Progress, token: vscode.CancellationToken): Thenable { + return new Promise((resolve, reject) => { + let isDone = false; + const cancel = () => { + this.isDone = true; + this.rgProc.kill(); + }; + token.onCancellationRequested(cancel); + + const rgArgs = getRgArgs(options); + + const cwd = options.folder.fsPath; + + // TODO logging + // const escapedArgs = rgArgs + // .map(arg => arg.match(/^-/) ? arg : `'${arg}'`) + // .join(' '); + // let rgCmd = `rg ${escapedArgs}\n - cwd: ${cwd}`; + + this.rgProc = cp.spawn(rgDiskPath, rgArgs, { cwd }); + process.once('exit', this.killRgProcFn); + this.rgProc.on('error', e => { + console.log(e); + reject(e); + }); + + let leftover = ''; + this.collectStdout(this.rgProc, (err, stdout, last) => { + if (err) { + reject(err); + return; + } + + // Mac: uses NFD unicode form on disk, but we want NFC + const normalized = leftover + (isMac ? normalizeNFC(stdout) : stdout); + const relativeFiles = normalized.split('\n'); + + if (last) { + const n = relativeFiles.length; + relativeFiles[n - 1] = relativeFiles[n - 1].trim(); + if (!relativeFiles[n - 1]) { + relativeFiles.pop(); + } + } else { + leftover = relativeFiles.pop(); + } + + if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) { + reject(new Error('Splitting up files failed')); + return; + } + + relativeFiles.forEach(relativeFile => { + progress.report(vscode.Uri.file(path.join(cwd, relativeFile))); + }); + + if (last) { + process.removeListener('exit', this.killRgProcFn); + if (isDone) { + resolve(); + } else { + // Trigger last result + this.rgProc = null; + if (err) { + reject(err); + } else { + resolve(); + } + } + } + }); + }); + } + + private collectStdout(cmd: cp.ChildProcess, cb: (err: Error, stdout?: string, last?: boolean) => void): void { + let done = (err: Error, stdout?: string, last?: boolean) => { + if (err || last) { + done = () => { }; + } + + cb(err, stdout, last); + }; + + this.forwardData(cmd.stdout, done); + const stderr = this.collectData(cmd.stderr); + + let gotData = false; + cmd.stdout.once('data', () => gotData = true); + + cmd.on('error', (err: Error) => { + done(err); + }); + + cmd.on('close', (code: number) => { + // ripgrep returns code=1 when no results are found + let stderrText, displayMsg: string; + if (!gotData && (stderrText = this.decodeData(stderr)) && (displayMsg = rgErrorMsgForDisplay(stderrText))) { + done(new Error(`command failed with error code ${code}: ${displayMsg}`)); + } else { + done(null, '', true); + } + }); + } + + private forwardData(stream: Readable, cb: (err: Error, stdout?: string) => void): NodeStringDecoder { + const decoder = new StringDecoder(); + stream.on('data', (data: Buffer) => { + cb(null, decoder.write(data)); + }); + return decoder; + } + + private collectData(stream: Readable): Buffer[] { + const buffers: Buffer[] = []; + stream.on('data', (data: Buffer) => { + buffers.push(data); + }); + return buffers; + } + + private decodeData(buffers: Buffer[]): string { + const decoder = new StringDecoder(); + return buffers.map(buffer => decoder.write(buffer)).join(''); + } +} + +function getRgArgs(options: vscode.FileSearchOptions): string[] { + const args = ['--files', '--hidden', '--case-sensitive']; + + options.includes.forEach(globArg => { + const inclusion = anchor(globArg); + args.push('-g', inclusion); + if (isMac) { + const normalized = normalizeNFD(inclusion); + if (normalized !== inclusion) { + args.push('-g', normalized); + } + } + }); + + options.excludes.forEach(globArg => { + const exclusion = `!${anchor(globArg)}`; + args.push('-g', exclusion); + if (isMac) { + const normalized = normalizeNFD(exclusion); + if (normalized !== exclusion) { + args.push('-g', normalized); + } + } + }); + + if (options.useIgnoreFiles) { + args.push('--no-ignore-parent'); + } else { + // Don't use .gitignore or .ignore + args.push('--no-ignore'); + } + + // Follow symlinks + if (options.followSymlinks) { + args.push('--follow'); + } + + // Folder to search + args.push('--'); + + args.push('.'); + + return args; +} + +function anchor(glob: string) { + return glob.startsWith('**') || glob.startsWith('/') ? glob : `/${glob}`; +} diff --git a/extensions/search-rg/src/ripgrepHelpers.ts b/extensions/search-rg/src/ripgrepHelpers.ts index 6949561a331..b982cf394da 100644 --- a/extensions/search-rg/src/ripgrepHelpers.ts +++ b/extensions/search-rg/src/ripgrepHelpers.ts @@ -9,17 +9,6 @@ import * as vscode from 'vscode'; import * as path from 'path'; -export function patternsToRgGlobs(patterns: vscode.GlobPattern[]): string[] { - return patterns.map(p => { - if (typeof p === 'string') { - return p; - } else { - // TODO - return p.pattern; - } - }); -} - export function fixDriveC(_path: string): string { const root = path.parse(_path).root; return root.toLowerCase() === 'c:/' ? diff --git a/extensions/search-rg/src/ripgrepTextSearch.ts b/extensions/search-rg/src/ripgrepTextSearch.ts index 415bb6f9ccc..99aa8262dc3 100644 --- a/extensions/search-rg/src/ripgrepTextSearch.ts +++ b/extensions/search-rg/src/ripgrepTextSearch.ts @@ -13,7 +13,6 @@ import { StringDecoder, NodeStringDecoder } from 'string_decoder'; import * as cp from 'child_process'; import { rgPath } from 'vscode-ripgrep'; -import { patternsToRgGlobs } from './ripgrepHelpers'; import { start } from 'repl'; // If vscode-ripgrep is in an .asar file, then the binary is unpacked. @@ -35,7 +34,6 @@ export class RipgrepTextSearchEngine { provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress, token: vscode.CancellationToken): Thenable { return new Promise((resolve, reject) => { - let isDone = false; const cancel = () => { this.isDone = true; this.ripgrepParser.cancel(); @@ -85,7 +83,7 @@ export class RipgrepTextSearchEngine { this.rgProc.on('close', code => { process.removeListener('exit', this.killRgProcFn); - if (isDone) { + if (this.isDone) { resolve(); } else { // Trigger last result @@ -304,10 +302,11 @@ function getRgArgs(query: vscode.TextSearchQuery, options: vscode.TextSearchOpti const args = ['--hidden', '--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold']; args.push(query.isCaseSensitive ? '--case-sensitive' : '--ignore-case'); - patternsToRgGlobs(options.includes) + // TODO@roblou + options.includes .forEach(globArg => args.push('-g', globArg)); - patternsToRgGlobs(options.excludes) + options.excludes .forEach(rgGlob => args.push('-g', `!${rgGlob}`)); if (options.maxFileSize) { diff --git a/src/vs/vscode.proposed.d.ts b/src/vs/vscode.proposed.d.ts index 06933b5f8e5..32be39d96c4 100644 --- a/src/vs/vscode.proposed.d.ts +++ b/src/vs/vscode.proposed.d.ts @@ -74,6 +74,8 @@ declare module 'vscode' { encoding?: string; } + export interface FileSearchOptions extends SearchOptions { } + export interface TextSearchResult { uri: Uri; range: Range; @@ -83,7 +85,7 @@ declare module 'vscode' { } export interface SearchProvider { - provideFileSearchResults?(query: string, options: SearchOptions, progress: Progress, token: CancellationToken): Thenable; + provideFileSearchResults?(options: FileSearchOptions, progress: Progress, token: CancellationToken): Thenable; provideTextSearchResults?(query: TextSearchQuery, options: TextSearchOptions, progress: Progress, token: CancellationToken): Thenable; } diff --git a/src/vs/workbench/api/electron-browser/mainThreadSearch.ts b/src/vs/workbench/api/electron-browser/mainThreadSearch.ts index e16a078ccc4..0249b46196c 100644 --- a/src/vs/workbench/api/electron-browser/mainThreadSearch.ts +++ b/src/vs/workbench/api/electron-browser/mainThreadSearch.ts @@ -108,7 +108,7 @@ class RemoteSearchProvider implements ISearchResultProvider { this._searches.set(search.id, search); outer = query.type === QueryType.File - ? this._proxy.$provideFileSearchResults(this._handle, search.id, query.filePattern) + ? this._proxy.$provideFileSearchResults(this._handle, search.id, query) : this._proxy.$provideTextSearchResults(this._handle, search.id, query.contentPattern, query); outer.then(() => { diff --git a/src/vs/workbench/api/node/extHost.protocol.ts b/src/vs/workbench/api/node/extHost.protocol.ts index a39da0025ef..0c0ca5ffc63 100644 --- a/src/vs/workbench/api/node/extHost.protocol.ts +++ b/src/vs/workbench/api/node/extHost.protocol.ts @@ -595,7 +595,7 @@ export interface ExtHostFileSystemShape { } export interface ExtHostSearchShape { - $provideFileSearchResults(handle: number, session: number, query: string): TPromise; + $provideFileSearchResults(handle: number, session: number, query: IRawSearchQuery): TPromise; $provideTextSearchResults(handle: number, session: number, pattern: IPatternInfo, query: IRawSearchQuery): TPromise; } diff --git a/src/vs/workbench/api/node/extHostSearch.ts b/src/vs/workbench/api/node/extHostSearch.ts index ef3f947e8ee..ea448f1960a 100644 --- a/src/vs/workbench/api/node/extHostSearch.ts +++ b/src/vs/workbench/api/node/extHostSearch.ts @@ -6,7 +6,7 @@ import { asWinJsPromise } from 'vs/base/common/async'; import { TPromise } from 'vs/base/common/winjs.base'; -import { IPatternInfo, IFolderQuery, IRawSearchQuery, IRawFileMatch, IFileMatch } from 'vs/platform/search/common/search'; +import { IPatternInfo, IFolderQuery, IRawSearchQuery, IRawFileMatch, IFileMatch, ISearchQuery } from 'vs/platform/search/common/search'; import * as vscode from 'vscode'; import { ExtHostSearchShape, IMainContext, MainContext, MainThreadSearchShape } from './extHost.protocol'; import URI, { UriComponents } from 'vs/base/common/uri'; @@ -33,17 +33,14 @@ export class ExtHostSearch implements ExtHostSearchShape { }; } - $provideFileSearchResults(handle: number, session: number, query: string): TPromise { - const provider = this._searchProvider.get(handle); - if (!provider.provideFileSearchResults) { - return TPromise.as(undefined); - } - const progress = { - report: (uri) => { - this._proxy.$handleFindMatch(handle, session, uri); - } - }; - return asWinJsPromise(token => provider.provideFileSearchResults(query, null, progress, token)); + $provideFileSearchResults(handle: number, session: number, query: IRawSearchQuery): TPromise { + return TPromise.join( + query.folderQueries.map(fq => this.provideFileSearchResultsForFolder(handle, session, query, fq)) + ).then( + () => { }, + (err: Error[]) => { + return TPromise.wrapError(err[0]); + }); } $provideTextSearchResults(handle: number, session: number, pattern: IPatternInfo, query: IRawSearchQuery): TPromise { @@ -56,6 +53,39 @@ export class ExtHostSearch implements ExtHostSearchShape { }); } + private provideFileSearchResultsForFolder(handle: number, session: number, query: IRawSearchQuery, folderQuery: IFolderQuery): TPromise { + const provider = this._searchProvider.get(handle); + if (!provider.provideFileSearchResults) { + return TPromise.as(undefined); + } + + const includes: string[] = query.includePattern ? Object.keys(query.includePattern) : []; + if (folderQuery.includePattern) { + includes.push(...Object.keys(folderQuery.includePattern)); + } + + const excludes: string[] = query.excludePattern ? Object.keys(query.excludePattern) : []; + if (folderQuery.excludePattern) { + excludes.push(...Object.keys(folderQuery.excludePattern)); + } + + const searchOptions: vscode.FileSearchOptions = { + folder: URI.from(folderQuery.folder), + excludes, + includes, + useIgnoreFiles: !query.disregardIgnoreFiles, + followSymlinks: !query.ignoreSymlinks + }; + + const progress = { + report: (data: vscode.Uri) => { + this._proxy.$handleFindMatch(handle, session, data); + } + }; + + return asWinJsPromise(token => provider.provideFileSearchResults(searchOptions, progress, token)); + } + private provideTextSearchResultsForFolder(handle: number, session: number, pattern: IPatternInfo, query: IRawSearchQuery, folderQuery: IFolderQuery): TPromise { const provider = this._searchProvider.get(handle); if (!provider.provideTextSearchResults) {