server - introduce support for nodejsRepository (#179105)

* server - introduce support for `nodejsRepository`

* fix path

* add auth

* proper download

* extract to utility function

* reuse

* fix compile

build/gulpfile.reh.js

@@ -17,6 +17,7 @@ const rename = require('gulp-rename');
 const replace = require('gulp-replace');
 const filter = require('gulp-filter');
 const { getProductionDependencies } = require('./lib/dependencies');
+const { assetFromGithub } = require('./lib/github');
 const vfs = require('vinyl-fs');
 const packageJson = require('../package.json');
 const flatmap = require('gulp-flatmap');
@@ -162,6 +163,11 @@ function nodejs(platform, arch) {
 	}
 
 	if (platform === 'win32') {
+		if (product.nodejsRepository) {
+			return assetFromGithub(product.nodejsRepository, nodeVersion, name => name === `win-${arch}-node.exe`)
+				.pipe(rename('node.exe'));
+		}
+
 		return remote(`/dist/v${nodeVersion}/win-${arch}/node.exe`, { base: 'https://nodejs.org' })
 			.pipe(rename('node.exe'));
 	}

build/lib/extensions.ts

@@ -9,8 +9,6 @@ import * as cp from 'child_process';
 import * as glob from 'glob';
 import * as gulp from 'gulp';
 import * as path from 'path';
-import * as through2 from 'through2';
-import got from 'got';
 import { Stream } from 'stream';
 import * as File from 'vinyl';
 import { createStatsStream } from './stats';
@@ -26,6 +24,7 @@ import webpack = require('webpack');
 import { getProductionDependencies } from './dependencies';
 import { getExtensionStream } from './builtInExtensions';
 import { getVersion } from './getVersion';
+import { assetFromGithub } from './github';
 
 const root = path.dirname(path.dirname(__dirname));
 const commit = getVersion(root);
@@ -238,39 +237,15 @@ export function fromMarketplace(serviceUrl: string, { name: extensionName, versi
 		.pipe(packageJsonFilter.restore);
 }
 
-const ghApiHeaders: Record<string, string> = {
-	Accept: 'application/vnd.github.v3+json',
-	'User-Agent': userAgent,
-};
-if (process.env.GITHUB_TOKEN) {
-	ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
-}
-const ghDownloadHeaders = {
-	...ghApiHeaders,
-	Accept: 'application/octet-stream',
-};
 
 export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream {
-	const remote = require('gulp-remote-retry-src');
 	const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
 
 	fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
 
 	const packageJsonFilter = filter('package.json', { restore: true });
 
-	return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
+	return assetFromGithub(new URL(repo).pathname, version, name => name.endsWith('.vsix'))
-		base: 'https://api.github.com',
-		requestOptions: { headers: ghApiHeaders }
-	}).pipe(through2.obj(function (file, _enc, callback) {
-		const asset = JSON.parse(file.contents.toString()).assets.find((a: any) => a.name.endsWith('.vsix'));
-		if (!asset) {
-			return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
-		}
-
-		const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
-		file.contents = res.pipe(through2());
-		callback(null, file);
-	}))
 		.pipe(buffer())
 		.pipe(vzip.src())
 		.pipe(filter('extension/**'))

build/lib/github.js

@@ -0,0 +1,43 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assetFromGithub = void 0;
+const got_1 = require("got");
+const remote = require("gulp-remote-retry-src");
+const through2 = require("through2");
+const ghApiHeaders = {
+    Accept: 'application/vnd.github.v3+json',
+    'User-Agent': 'VSCode Build',
+};
+if (process.env.GITHUB_TOKEN) {
+    ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
+}
+const ghDownloadHeaders = {
+    ...ghApiHeaders,
+    Accept: 'application/octet-stream',
+};
+/**
+ * @param repo for example `Microsoft/vscode`
+ * @param version for example `16.17.1` - must be a valid releases tag
+ * @param assetName for example (name) => name === `win-x64-node.exe` - must be an asset that exists
+ * @returns a stream with the asset as file
+ */
+function assetFromGithub(repo, version, assetFilter) {
+    return remote(`/repos/${repo.replace(/^\/|\/$/g, '')}/releases/tags/v${version}`, {
+        base: 'https://api.github.com',
+        requestOptions: { headers: ghApiHeaders }
+    }).pipe(through2.obj(function (file, _enc, callback) {
+        const asset = JSON.parse(file.contents.toString()).assets.find((a) => assetFilter(a.name));
+        if (!asset) {
+            return callback(new Error(`Could not find asset in release of ${repo} @ ${version}`));
+        }
+        const res = got_1.default.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
+        file.contents = res.pipe(through2());
+        callback(null, file);
+    }));
+}
+exports.assetFromGithub = assetFromGithub;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2l0aHViLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZ2l0aHViLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7O2dHQUdnRzs7O0FBR2hHLDZCQUFzQjtBQUN0QixnREFBZ0Q7QUFDaEQscUNBQXFDO0FBRXJDLE1BQU0sWUFBWSxHQUEyQjtJQUM1QyxNQUFNLEVBQUUsZ0NBQWdDO0lBQ3hDLFlBQVksRUFBRSxjQUFjO0NBQzVCLENBQUM7QUFDRixJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsWUFBWSxFQUFFO0lBQzdCLFlBQVksQ0FBQyxhQUFhLEdBQUcsUUFBUSxHQUFHLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLENBQUM7Q0FDakc7QUFDRCxNQUFNLGlCQUFpQixHQUFHO0lBQ3pCLEdBQUcsWUFBWTtJQUNmLE1BQU0sRUFBRSwwQkFBMEI7Q0FDbEMsQ0FBQztBQUVGOzs7OztHQUtHO0FBQ0gsU0FBZ0IsZUFBZSxDQUFDLElBQVksRUFBRSxPQUFlLEVBQUUsV0FBc0M7SUFDcEcsT0FBTyxNQUFNLENBQUMsVUFBVSxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsRUFBRSxFQUFFLENBQUMsbUJBQW1CLE9BQU8sRUFBRSxFQUFFO1FBQ2pGLElBQUksRUFBRSx3QkFBd0I7UUFDOUIsY0FBYyxFQUFFLEVBQUUsT0FBTyxFQUFFLFlBQVksRUFBRTtLQUN6QyxDQUFDLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsVUFBVSxJQUFJLEVBQUUsSUFBSSxFQUFFLFFBQVE7UUFDbEQsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQW1CLEVBQUUsRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztRQUM3RyxJQUFJLENBQUMsS0FBSyxFQUFFO1lBQ1gsT0FBTyxRQUFRLENBQUMsSUFBSSxLQUFLLENBQUMsc0NBQXNDLElBQUksTUFBTSxPQUFPLEVBQUUsQ0FBQyxDQUFDLENBQUM7U0FDdEY7UUFFRCxNQUFNLEdBQUcsR0FBRyxhQUFHLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUUsRUFBRSxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsY0FBYyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7UUFDeEYsSUFBSSxDQUFDLFFBQVEsR0FBRyxHQUFHLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7UUFDckMsUUFBUSxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQztJQUN0QixDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ0wsQ0FBQztBQWRELDBDQWNDIn0=

build/lib/github.ts

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Stream } from 'stream';
+import got from 'got';
+import * as remote from 'gulp-remote-retry-src';
+import * as through2 from 'through2';
+
+const ghApiHeaders: Record<string, string> = {
+	Accept: 'application/vnd.github.v3+json',
+	'User-Agent': 'VSCode Build',
+};
+if (process.env.GITHUB_TOKEN) {
+	ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
+}
+const ghDownloadHeaders = {
+	...ghApiHeaders,
+	Accept: 'application/octet-stream',
+};
+
+/**
+ * @param repo for example `Microsoft/vscode`
+ * @param version for example `16.17.1` - must be a valid releases tag
+ * @param assetName for example (name) => name === `win-x64-node.exe` - must be an asset that exists
+ * @returns a stream with the asset as file
+ */
+export function assetFromGithub(repo: string, version: string, assetFilter: (name: string) => boolean): Stream {
+	return remote(`/repos/${repo.replace(/^\/|\/$/g, '')}/releases/tags/v${version}`, {
+		base: 'https://api.github.com',
+		requestOptions: { headers: ghApiHeaders }
+	}).pipe(through2.obj(function (file, _enc, callback) {
+		const asset = JSON.parse(file.contents.toString()).assets.find((a: { name: string }) => assetFilter(a.name));
+		if (!asset) {
+			return callback(new Error(`Could not find asset in release of ${repo} @ ${version}`));
+		}
+
+		const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
+		file.contents = res.pipe(through2());
+		callback(null, file);
+	}));
+}

package.json

@@ -1,7 +1,7 @@
 {
   "name": "code-oss-dev",
   "version": "1.78.0",
-  "distro": "41a18d6ff0c57bb1e805906a181778fe4747cf86",
+  "distro": "8e1c873581aa258fc90ddfa322f0d76e49f1c822",
   "author": {
     "name": "Microsoft Corporation"
   },