mirror of
https://github.com/microsoft/vscode.git
synced 2026-05-16 21:31:01 +01:00
Jeff Hunter
75 lines
2.3 KiB
TypeScript
75 lines
2.3 KiB
TypeScript
/*---------------------------------------------------------------------------------------------
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
*--------------------------------------------------------------------------------------------*/
|
|
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
|
|
async function copyStaticAssets(srcpaths: string[], dst: string): Promise<void> {
|
|
await Promise.all(srcpaths.map(async srcpath => {
|
|
const src = path.join(REPO_ROOT, srcpath);
|
|
const dest = path.join(REPO_ROOT, dst, path.basename(srcpath));
|
|
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
|
await fs.promises.copyFile(src, dest);
|
|
}));
|
|
}
|
|
|
|
async function fileExists(filePath: string): Promise<boolean> {
|
|
try {
|
|
await fs.promises.access(filePath, fs.constants.F_OK);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
const treeSitterGrammars: string[] = [
|
|
'tree-sitter-c-sharp',
|
|
'tree-sitter-cpp',
|
|
'tree-sitter-go',
|
|
'tree-sitter-javascript', // Also includes jsx support
|
|
'tree-sitter-python',
|
|
'tree-sitter-ruby',
|
|
'tree-sitter-typescript',
|
|
'tree-sitter-tsx',
|
|
'tree-sitter-java',
|
|
'tree-sitter-rust',
|
|
'tree-sitter-php'
|
|
];
|
|
|
|
const REPO_ROOT = path.join(__dirname, '..');
|
|
|
|
async function platformDir(): Promise<string> {
|
|
const distPath = 'dist/src/_internal/platform';
|
|
const srcPath = 'src/_internal/platform';
|
|
if (await fileExists(path.join(REPO_ROOT, distPath))) {
|
|
return distPath;
|
|
} else if (await fileExists(path.join(REPO_ROOT, srcPath))) {
|
|
return srcPath;
|
|
} else {
|
|
throw new Error('Could not find the source directory for tokenizer files');
|
|
}
|
|
}
|
|
|
|
function treeSitterWasmDir(): string {
|
|
const modulePath = path.dirname(require.resolve('@vscode/tree-sitter-wasm'));
|
|
return path.relative(REPO_ROOT, modulePath);
|
|
}
|
|
|
|
async function main() {
|
|
const platform = await platformDir();
|
|
const vendoredTiktokenFiles = [`${platform}/tokenizer/node/cl100k_base.tiktoken`, `${platform}/tokenizer/node/o200k_base.tiktoken`];
|
|
const wasm = treeSitterWasmDir();
|
|
|
|
// copy static assets to dist
|
|
await copyStaticAssets([
|
|
...vendoredTiktokenFiles,
|
|
...treeSitterGrammars.map(grammar => `${wasm}/${grammar}.wasm`),
|
|
`${wasm}/tree-sitter.wasm`,
|
|
], 'dist');
|
|
|
|
}
|
|
|
|
main();
|