From ca48c64699f221b18e487875b782f97c7523c7cd Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Tue, 2 Aug 2022 15:27:16 -0700 Subject: [PATCH] build: cache built-in extensions to avoid rate limiting (#156918) --- build/azure-pipelines/product-compile.yml | 12 ++++++++++++ build/gulpfile.extensions.js | 2 +- build/gulpfile.vscode.web.js | 2 +- build/lib/builtInExtensions.js | 20 +++++++++++++++++--- build/lib/builtInExtensions.ts | 21 ++++++++++++++++++--- build/lib/extensions.js | 8 ++++---- build/lib/extensions.ts | 8 ++++---- 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index 381d49ee75a..7df5d1505f2 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -56,6 +56,13 @@ steps: cacheHitVar: NODE_MODULES_RESTORED displayName: Restore node_modules cache + # Cache built-in extensions to avoid GH rate limits. + - task: Cache@2 + inputs: + key: '"$(Agent.OS)" | product.json' + path: .build/builtInExtensions + displayName: Restore built-in extensions + - script: | set -e tar -xzf .build/node_modules_cache/cache.tgz @@ -109,6 +116,11 @@ steps: node build/azure-pipelines/mixin displayName: Mix in quality + - script: | + set -e + node build/lib/builtInExtensions.js + displayName: Download missing built-in extensions + - script: | set -e yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check diff --git a/build/gulpfile.extensions.js b/build/gulpfile.extensions.js index bb893f02923..04132cd4400 100644 --- a/build/gulpfile.extensions.js +++ b/build/gulpfile.extensions.js @@ -236,8 +236,8 @@ exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask; const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions')); const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series( cleanExtensionsBuildTask, + task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))), task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))), - task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build'))), )); gulp.task(compileExtensionsBuildTask); diff --git a/build/gulpfile.vscode.web.js b/build/gulpfile.vscode.web.js index 4c1259c241c..56b08474033 100644 --- a/build/gulpfile.vscode.web.js +++ b/build/gulpfile.vscode.web.js @@ -229,7 +229,7 @@ function packageTask(sourceFolderName, destinationFolderName) { const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series( task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')), task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true).pipe(gulp.dest('.build/web'))), - task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build/web'))), + task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))), task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')), )); gulp.task(compileWebExtensionsBuildTask); diff --git a/build/lib/builtInExtensions.js b/build/lib/builtInExtensions.js index f38871c36d7..38c30234b7e 100644 --- a/build/lib/builtInExtensions.js +++ b/build/lib/builtInExtensions.js @@ -4,7 +4,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.getBuiltInExtensions = void 0; +exports.getBuiltInExtensions = exports.getExtensionStream = void 0; const fs = require("fs"); const path = require("path"); const os = require("os"); @@ -44,6 +44,21 @@ function isUpToDate(extension) { return false; } } +function getExtensionDownloadStream(extension) { + const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; + return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) + .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)); +} +function getExtensionStream(extension) { + // if the extension exists on disk, use those files instead of downloading anew + if (isUpToDate(extension)) { + log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎')); + return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true }) + .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)); + } + return getExtensionDownloadStream(extension); +} +exports.getExtensionStream = getExtensionStream; function syncMarketplaceExtension(extension) { const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]'); @@ -52,8 +67,7 @@ function syncMarketplaceExtension(extension) { return es.readArray([]); } rimraf.sync(getExtensionPath(extension)); - return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) - .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) + return getExtensionDownloadStream(extension) .pipe(vfs.dest('.build/builtInExtensions')) .on('end', () => log(source, extension.name, ansiColors.green('✔︎'))); } diff --git a/build/lib/builtInExtensions.ts b/build/lib/builtInExtensions.ts index 971847c8756..912e05653ac 100644 --- a/build/lib/builtInExtensions.ts +++ b/build/lib/builtInExtensions.ts @@ -68,10 +68,26 @@ function isUpToDate(extension: IExtensionDefinition): boolean { } } +function getExtensionDownloadStream(extension: IExtensionDefinition) { + const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; + return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) + .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)); +} + +export function getExtensionStream(extension: IExtensionDefinition) { + // if the extension exists on disk, use those files instead of downloading anew + if (isUpToDate(extension)) { + log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎')); + return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true }) + .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)); + } + + return getExtensionDownloadStream(extension); +} + function syncMarketplaceExtension(extension: IExtensionDefinition): Stream { const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]'); - if (isUpToDate(extension)) { log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); return es.readArray([]); @@ -79,8 +95,7 @@ function syncMarketplaceExtension(extension: IExtensionDefinition): Stream { rimraf.sync(getExtensionPath(extension)); - return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) - .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) + return getExtensionDownloadStream(extension) .pipe(vfs.dest('.build/builtInExtensions')) .on('end', () => log(source, extension.name, ansiColors.green('✔︎'))); } diff --git a/build/lib/extensions.js b/build/lib/extensions.js index 808d817b815..009bbe4b014 100644 --- a/build/lib/extensions.js +++ b/build/lib/extensions.js @@ -25,6 +25,7 @@ const buffer = require('gulp-buffer'); const jsoncParser = require("jsonc-parser"); const dependencies_1 = require("./dependencies"); const _ = require("underscore"); +const builtInExtensions_1 = require("./builtInExtensions"); const util = require('./util'); const root = path.dirname(path.dirname(__dirname)); const commit = util.getVersion(root); @@ -312,16 +313,15 @@ function packageLocalExtensionsStream(forWeb) { .pipe(util2.setExecutableBit(['**/*.sh']))); } exports.packageLocalExtensionsStream = packageLocalExtensionsStream; -function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) { +function packageMarketplaceExtensionsStream(forWeb) { const marketplaceExtensionsDescriptions = [ ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...(forWeb ? webBuiltInExtensions : []) ]; const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions .map(extension => { - const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension)) - .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); - return updateExtensionPackageJSON(input, (data) => { + const src = (0, builtInExtensions_1.getExtensionStream)(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`)); + return updateExtensionPackageJSON(src, (data) => { delete data.scripts; delete data.dependencies; delete data.devDependencies; diff --git a/build/lib/extensions.ts b/build/lib/extensions.ts index ddcb25483fb..d14ddca2f8a 100644 --- a/build/lib/extensions.ts +++ b/build/lib/extensions.ts @@ -25,6 +25,7 @@ import * as jsoncParser from 'jsonc-parser'; import webpack = require('webpack'); import { getProductionDependencies } from './dependencies'; import _ = require('underscore'); +import { getExtensionStream } from './builtInExtensions'; const util = require('./util'); const root = path.dirname(path.dirname(__dirname)); const commit = util.getVersion(root); @@ -381,7 +382,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream { ); } -export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream { +export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream { const marketplaceExtensionsDescriptions = [ ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...(forWeb ? webBuiltInExtensions : []) @@ -390,9 +391,8 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServi es.merge( ...marketplaceExtensionsDescriptions .map(extension => { - const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension)) - .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); - return updateExtensionPackageJSON(input, (data: any) => { + const src = getExtensionStream(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`)); + return updateExtensionPackageJSON(src, (data: any) => { delete data.scripts; delete data.dependencies; delete data.devDependencies;