From 72d9aa625e8488c47fa40404b4dbdc140f1f3a87 Mon Sep 17 00:00:00 2001 From: Nicholas Rayburn <52075362+nrayburn-tech@users.noreply.github.com> Date: Mon, 2 Aug 2021 21:44:56 -0500 Subject: [PATCH] Fix #124276 batch markdown file requests (#124545) * fix #124276 batch markdown file requests * cleanup and modify return type * Revert "cleanup and modify return type" This reverts commit 62d62b4947b83fd2ec488c448c2f4eed4e5cdbf5. * cleanup getAllMarkdownDocuments * fix markdown batching * fix var not being modified cleanup function call remove modulo use * don't create a new array on each iteration Co-authored-by: Matt Bierner --- .../src/features/workspaceSymbolProvider.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/extensions/markdown-language-features/src/features/workspaceSymbolProvider.ts b/extensions/markdown-language-features/src/features/workspaceSymbolProvider.ts index 3a9624a42f5..48ecd330178 100644 --- a/extensions/markdown-language-features/src/features/workspaceSymbolProvider.ts +++ b/extensions/markdown-language-features/src/features/workspaceSymbolProvider.ts @@ -28,10 +28,23 @@ class VSCodeWorkspaceMarkdownDocumentProvider extends Disposable implements Work private readonly utf8Decoder = new TextDecoder('utf-8'); - async getAllMarkdownDocuments() { + /** + * Reads and parses all .md documents in the workspace. + * Files are processed in batches, to keep the number of open files small. + * + * @returns Array of processed .md files. + */ + async getAllMarkdownDocuments(): Promise { + const maxConcurrent = 20; + const docList: SkinnyTextDocument[] = []; const resources = await vscode.workspace.findFiles('**/*.md', '**/node_modules/**'); - const docs = await Promise.all(resources.map(doc => this.getMarkdownDocument(doc))); - return docs.filter(doc => !!doc) as SkinnyTextDocument[]; + + for (let i = 0; i < resources.length; i += maxConcurrent) { + const resourceBatch = resources.slice(i, i + maxConcurrent); + const documentBatch = (await Promise.all(resourceBatch.map(this.getMarkdownDocument))).filter((doc) => !!doc) as SkinnyTextDocument[]; + docList.push(...documentBatch); + } + return docList; } public get onDidChangeMarkdownDocument() {