build: cache built-in extensions to avoid rate limiting (#156918)

build/azure-pipelines/product-compile.yml

@@ -56,6 +56,13 @@ steps:
       cacheHitVar: NODE_MODULES_RESTORED
     displayName: Restore node_modules cache
 
+  # Cache built-in extensions to avoid GH rate limits.
+  - task: Cache@2
+    inputs:
+      key: '"$(Agent.OS)" | product.json'
+      path: .build/builtInExtensions
+    displayName: Restore built-in extensions
+
   - script: |
       set -e
       tar -xzf .build/node_modules_cache/cache.tgz
@@ -109,6 +116,11 @@ steps:
         node build/azure-pipelines/mixin
       displayName: Mix in quality
 
+  - script: |
+      set -e
+      node build/lib/builtInExtensions.js
+    displayName: Download missing built-in extensions
+
   - script: |
       set -e
       yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check

build/gulpfile.extensions.js

@@ -236,8 +236,8 @@ exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;
 const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions'));
 const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
 	cleanExtensionsBuildTask,
+	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
 	task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))),
-	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build'))),
 ));
 
 gulp.task(compileExtensionsBuildTask);

build/gulpfile.vscode.web.js

@@ -229,7 +229,7 @@ function packageTask(sourceFolderName, destinationFolderName) {
 const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series(
 	task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')),
 	task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true).pipe(gulp.dest('.build/web'))),
-	task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build/web'))),
+	task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))),
 	task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')),
 ));
 gulp.task(compileWebExtensionsBuildTask);

build/lib/builtInExtensions.js

@@ -4,7 +4,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBuiltInExtensions = void 0;
+exports.getBuiltInExtensions = exports.getExtensionStream = void 0;
 const fs = require("fs");
 const path = require("path");
 const os = require("os");
@@ -44,6 +44,21 @@ function isUpToDate(extension) {
         return false;
     }
 }
+function getExtensionDownloadStream(extension) {
+    const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
+    return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
+        .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
+}
+function getExtensionStream(extension) {
+    // if the extension exists on disk, use those files instead of downloading anew
+    if (isUpToDate(extension)) {
+        log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎'));
+        return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true })
+            .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
+    }
+    return getExtensionDownloadStream(extension);
+}
+exports.getExtensionStream = getExtensionStream;
 function syncMarketplaceExtension(extension) {
     const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
     const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
@@ -52,8 +67,7 @@ function syncMarketplaceExtension(extension) {
         return es.readArray([]);
     }
     rimraf.sync(getExtensionPath(extension));
-    return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
+    return getExtensionDownloadStream(extension)
-        .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
         .pipe(vfs.dest('.build/builtInExtensions'))
         .on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
 }

build/lib/builtInExtensions.ts

@@ -68,10 +68,26 @@ function isUpToDate(extension: IExtensionDefinition): boolean {
 	}
 }
 
+function getExtensionDownloadStream(extension: IExtensionDefinition) {
+	const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
+	return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
+		.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
+}
+
+export function getExtensionStream(extension: IExtensionDefinition) {
+	// if the extension exists on disk, use those files instead of downloading anew
+	if (isUpToDate(extension)) {
+		log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎'));
+		return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true })
+			.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
+	}
+
+	return getExtensionDownloadStream(extension);
+}
+
 function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
 	const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
 	const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
-
 	if (isUpToDate(extension)) {
 		log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
 		return es.readArray([]);
@@ -79,8 +95,7 @@ function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
 
 	rimraf.sync(getExtensionPath(extension));
 
-	return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
+	return getExtensionDownloadStream(extension)
-		.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
 		.pipe(vfs.dest('.build/builtInExtensions'))
 		.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
 }

build/lib/extensions.js

@@ -25,6 +25,7 @@ const buffer = require('gulp-buffer');
 const jsoncParser = require("jsonc-parser");
 const dependencies_1 = require("./dependencies");
 const _ = require("underscore");
+const builtInExtensions_1 = require("./builtInExtensions");
 const util = require('./util');
 const root = path.dirname(path.dirname(__dirname));
 const commit = util.getVersion(root);
@@ -312,16 +313,15 @@ function packageLocalExtensionsStream(forWeb) {
         .pipe(util2.setExecutableBit(['**/*.sh'])));
 }
 exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
-function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) {
+function packageMarketplaceExtensionsStream(forWeb) {
     const marketplaceExtensionsDescriptions = [
         ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
         ...(forWeb ? webBuiltInExtensions : [])
     ];
     const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions
         .map(extension => {
-        const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
+        const src = (0, builtInExtensions_1.getExtensionStream)(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`));
-            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+        return updateExtensionPackageJSON(src, (data) => {
-        return updateExtensionPackageJSON(input, (data) => {
             delete data.scripts;
             delete data.dependencies;
             delete data.devDependencies;

build/lib/extensions.ts

@@ -25,6 +25,7 @@ import * as jsoncParser from 'jsonc-parser';
 import webpack = require('webpack');
 import { getProductionDependencies } from './dependencies';
 import _ = require('underscore');
+import { getExtensionStream } from './builtInExtensions';
 const util = require('./util');
 const root = path.dirname(path.dirname(__dirname));
 const commit = util.getVersion(root);
@@ -381,7 +382,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
 	);
 }
 
-export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream {
+export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
 	const marketplaceExtensionsDescriptions = [
 		...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
 		...(forWeb ? webBuiltInExtensions : [])
@@ -390,9 +391,8 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServi
 		es.merge(
 			...marketplaceExtensionsDescriptions
 				.map(extension => {
-					const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
+					const src = getExtensionStream(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`));
-						.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+					return updateExtensionPackageJSON(src, (data: any) => {
-					return updateExtensionPackageJSON(input, (data: any) => {
 						delete data.scripts;
 						delete data.dependencies;
 						delete data.devDependencies;