Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-05-08 09:08:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

cleanup node tasks

Browse Source
This commit is contained in:
Joao Moreno
2019-07-01 15:20:38 +02:00
parent 2f1373c0bb
commit 9d75c4e528
5 changed files with 55 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/download/download.js
-91
View File
@@ -1,91 +0,0 @@
"use strict";
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
const https = require("https");
const fs = require("fs");
const path = require("path");
const cp = require("child_process");
function ensureDir(filepath) {
if (!fs.existsSync(filepath)) {
ensureDir(path.dirname(filepath));
fs.mkdirSync(filepath);
}
}
function download(options, destination) {
ensureDir(path.dirname(destination));
return new Promise((c, e) => {
const fd = fs.openSync(destination, 'w');
const req = https.get(options, (res) => {
res.on('data', (chunk) => {
fs.writeSync(fd, chunk);
});
res.on('end', () => {
fs.closeSync(fd);
c();
});
});
req.on('error', (reqErr) => {
console.error(`request to ${options.host}${options.path} failed.`);
console.error(reqErr);
e(reqErr);
});
});
}
const MARKER_ARGUMENT = `_download_fork_`;
function base64encode(str) {
return Buffer.from(str, 'utf8').toString('base64');
}
function base64decode(str) {
return Buffer.from(str, 'base64').toString('utf8');
}
function downloadInExternalProcess(options) {
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
console.log(`Downloading ${url}...`);
return new Promise((c, e) => {
const child = cp.fork(__filename, [MARKER_ARGUMENT, base64encode(JSON.stringify(options))], {
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
});
let stderr = [];
child.stderr.on('data', (chunk) => {
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
});
child.on('exit', (code) => {
if (code === 0) {
// normal termination
console.log(`Finished downloading ${url}.`);
c();
}
else {
// abnormal termination
console.error(Buffer.concat(stderr).toString());
e(new Error(`Download of ${url} failed.`));
}
});
});
}
exports.downloadInExternalProcess = downloadInExternalProcess;
function _downloadInExternalProcess() {
let options;
try {
options = JSON.parse(base64decode(process.argv[3]));
}
catch (err) {
console.error(`Cannot read arguments`);
console.error(err);
process.exit(-1);
return;
}
download(options.requestOptions, options.destinationPath).then(() => {
process.exit(0);
}, (err) => {
console.error(err);
process.exit(-2);
});
}
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
// running as forked download script
_downloadInExternalProcess();
}
build/download/download.ts
-111
View File
@@ -1,111 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as https from 'https';
import * as fs from 'fs';
import * as path from 'path';
import * as cp from 'child_process';
function ensureDir(filepath: string) {
if (!fs.existsSync(filepath)) {
ensureDir(path.dirname(filepath));
fs.mkdirSync(filepath);
}
}
function download(options: https.RequestOptions, destination: string): Promise<void> {
ensureDir(path.dirname(destination));
return new Promise<void>((c, e) => {
const fd = fs.openSync(destination, 'w');
const req = https.get(options, (res) => {
res.on('data', (chunk) => {
fs.writeSync(fd, chunk);
});
res.on('end', () => {
fs.closeSync(fd);
c();
});
});
req.on('error', (reqErr) => {
console.error(`request to ${options.host}${options.path} failed.`);
console.error(reqErr);
e(reqErr);
});
});
}
const MARKER_ARGUMENT = `_download_fork_`;
function base64encode(str: string): string {
return Buffer.from(str, 'utf8').toString('base64');
}
function base64decode(str: string): string {
return Buffer.from(str, 'base64').toString('utf8');
}
export interface IDownloadRequestOptions {
host: string;
path: string;
}
export interface IDownloadOptions {
requestOptions: IDownloadRequestOptions;
destinationPath: string;
}
export function downloadInExternalProcess(options: IDownloadOptions): Promise<void> {
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
console.log(`Downloading ${url}...`);
return new Promise<void>((c, e) => {
const child = cp.fork(
__filename,
[MARKER_ARGUMENT, base64encode(JSON.stringify(options))],
{
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
}
);
let stderr: Buffer[] = [];
child.stderr.on('data', (chunk) => {
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
});
child.on('exit', (code) => {
if (code === 0) {
// normal termination
console.log(`Finished downloading ${url}.`);
c();
} else {
// abnormal termination
console.error(Buffer.concat(stderr).toString());
e(new Error(`Download of ${url} failed.`));
}
});
});
}
function _downloadInExternalProcess() {
let options: IDownloadOptions;
try {
options = JSON.parse(base64decode(process.argv[3]));
} catch (err) {
console.error(`Cannot read arguments`);
console.error(err);
process.exit(-1);
return;
}
download(options.requestOptions, options.destinationPath).then(() => {
process.exit(0);
}, (err) => {
console.error(err);
process.exit(-2);
});
}
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
// running as forked download script
_downloadInExternalProcess();
}
build/gulpfile.reh.js
+49 -71
View File
@@ -11,18 +11,29 @@ const path = require('path');
const es = require('event-stream'); const es = require('event-stream');
const util = require('./lib/util'); const util = require('./lib/util');
const task = require('./lib/task'); const task = require('./lib/task');
const vfs = require('vinyl-fs'); const vfs = require('vinyl-fs');
const flatmap = require('gulp-flatmap'); const flatmap = require('gulp-flatmap');
const gunzip = require('gulp-gunzip'); const gunzip = require('gulp-gunzip');
const untar = require('gulp-untar'); const untar = require('gulp-untar');
const File = require('vinyl'); const File = require('vinyl');
const fs = require('fs'); const fs = require('fs');
const remote = require('gulp-remote-src');
const rename = require('gulp-rename');
const filter = require('gulp-filter');
const cp = require('child_process'); const cp = require('child_process');
const REPO_ROOT = path.dirname(__dirname); const REPO_ROOT = path.dirname(__dirname);
const BUILD_TARGETS = [
{ platform: 'win32', arch: 'ia32', pkgTarget: 'node8-win-x86' },
{ platform: 'win32', arch: 'x64', pkgTarget: 'node8-win-x64' },
{ platform: 'darwin', arch: null, pkgTarget: 'node8-macos-x64' },
{ platform: 'linux', arch: 'ia32', pkgTarget: 'node8-linux-x86' },
{ platform: 'linux', arch: 'x64', pkgTarget: 'node8-linux-x64' },
{ platform: 'linux', arch: 'armhf', pkgTarget: 'node8-linux-armv7' },
{ platform: 'linux', arch: 'alpine', pkgTarget: 'node8-linux-alpine' },
];
const noop = () => { return Promise.resolve(); }; const noop = () => { return Promise.resolve(); };
gulp.task('vscode-reh-win32-ia32-min', noop); gulp.task('vscode-reh-win32-ia32-min', noop);
@@ -39,64 +50,48 @@ function getNodeVersion() {
return target; return target;
} }
function ensureDirs(dirPath) { const nodeVersion = getNodeVersion();
if (!fs.existsSync(dirPath)) {
ensureDirs(path.dirname(dirPath)); BUILD_TARGETS.forEach(({ platform, arch }) => {
fs.mkdirSync(dirPath); const target = arch ? `${platform}-${arch}` : platform;
}
gulp.task(task.define(`node-${target}`, () => {
if (platform === 'darwin') {
arch = 'x64';
}
const nodePath = path.join('.build', 'node', `v${nodeVersion}`, `${platform}-${arch}`);
if (!fs.existsSync(nodePath)) {
util.rimraf(nodePath);
return nodejs(platform, arch)
.pipe(vfs.dest(nodePath));
}
return Promise.resolve(null);
}));
});
const defaultNodeTask = gulp.task(`node-${process.platform}-${process.arch}`);
if (defaultNodeTask) {
gulp.task(task.define('node', defaultNodeTask));
} }
/* Downloads the node executable used for the remote server to ./build/node-remote */
gulp.task(task.define('node-remote', () => {
const VERSION = getNodeVersion();
const nodePath = path.join('.build', 'node-remote');
const nodeVersionPath = path.join(nodePath, 'version');
if (!fs.existsSync(nodeVersionPath) || fs.readFileSync(nodeVersionPath).toString() !== VERSION) {
ensureDirs(nodePath);
util.rimraf(nodePath);
fs.writeFileSync(nodeVersionPath, VERSION);
return nodejs(process.platform, process.arch).pipe(vfs.dest(nodePath));
}
return vfs.src(nodePath);
}));
function nodejs(platform, arch) { function nodejs(platform, arch) {
const VERSION = getNodeVersion();
if (arch === 'ia32') { if (arch === 'ia32') {
arch = 'x86'; arch = 'x86';
} }
if (platform === 'win32') { if (platform === 'win32') {
const downloadPath = `/dist/v${VERSION}/win-${arch}/node.exe`; return remote(`/dist/v${nodeVersion}/win-${arch}/node.exe`, { base: 'https://nodejs.org' })
.pipe(rename('node.exe'));
return (
util.download({ host: 'nodejs.org', path: downloadPath })
.pipe(es.through(function (data) {
// base comes in looking like `https:\nodejs.org\dist\v10.2.1\win-x64\node.exe`
this.emit('data', new File({
path: data.path,
base: data.base.replace(/\\node\.exe$/, ''),
contents: data.contents,
stat: {
isFile: true,
mode: /* 100755 */ 33261
}
}));
}))
);
} }
if (arch === 'alpine') { if (arch === 'alpine') {
return es.readArray([ const contents = cp.execSync(`docker run --rm node:${nodeVersion}-alpine /bin/sh -c 'cat \`which node\`'`, { maxBuffer: 100 * 1024 * 1024, encoding: 'buffer' });
new File({ return es.readArray([new File({ path: 'node', contents, stat: { mode: parseInt('755', 8) } })]);
path: 'node',
contents: cp.execSync(`docker run --rm node:${VERSION}-alpine /bin/sh -c 'cat \`which node\`'`, { maxBuffer: 100 * 1024 * 1024, encoding: 'buffer' }),
stat: {
mode: parseInt('755', 8)
}
})
]);
} }
if (platform === 'darwin') { if (platform === 'darwin') {
@@ -107,28 +102,11 @@ function nodejs(platform, arch) {
arch = 'armv7l'; arch = 'armv7l';
} }
const downloadPath = `/dist/v${VERSION}/node-v${VERSION}-${platform}-${arch}.tar.gz`; return remote(`/dist/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: 'https://nodejs.org' })
.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
return ( .pipe(filter('**/node'))
util.download({ host: 'nodejs.org', path: downloadPath }) .pipe(util.setExecutableBit('**'))
.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar()))) .pipe(rename('node'));
.pipe(es.through(function (data) {
// base comes in looking like `https:/nodejs.org/dist/v8.9.3/node-v8.9.3-darwin-x64.tar.gz`
// => we must remove the `.tar.gz`
// Also, keep only bin/node
if (/\/bin\/node$/.test(data.path)) {
this.emit('data', new File({
path: data.path.replace(/bin\/node$/, 'node'),
base: data.base.replace(/\.tar\.gz$/, ''),
contents: data.contents,
stat: {
isFile: true,
mode: /* 100755 */ 33261
}
}));
}
}))
);
} }
function mixinServer(watch) { function mixinServer(watch) {
build/lib/util.js
+3 -37
View File
@@ -13,8 +13,6 @@ const fs = require("fs");
const _rimraf = require("rimraf"); const _rimraf = require("rimraf");
const git = require("./git"); const git = require("./git");
const VinylFile = require("vinyl"); const VinylFile = require("vinyl");
const download_1 = require("../download/download");
const REPO_ROOT = path.join(__dirname, '../../');
const NoCancellationToken = { isCancellationRequested: () => false }; const NoCancellationToken = { isCancellationRequested: () => false };
function incremental(streamProvider, initial, supportsCancellation) { function incremental(streamProvider, initial, supportsCancellation) {
const input = es.through(); const input = es.through();
@@ -68,6 +66,9 @@ function fixWin32DirectoryPermissions() {
exports.fixWin32DirectoryPermissions = fixWin32DirectoryPermissions; exports.fixWin32DirectoryPermissions = fixWin32DirectoryPermissions;
function setExecutableBit(pattern) { function setExecutableBit(pattern) {
const setBit = es.mapSync(f => { const setBit = es.mapSync(f => {
if (!f.stat) {
f.stat = { isFile() { return true; } };
}
f.stat.mode = /* 100755 */ 33261; f.stat.mode = /* 100755 */ 33261;
return f; return f;
}); });
@@ -218,38 +219,3 @@ function versionStringToNumber(versionStr) {
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10); return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
} }
exports.versionStringToNumber = versionStringToNumber; exports.versionStringToNumber = versionStringToNumber;
function download(requestOptions) {
const result = es.through();
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
const opts = {
requestOptions: requestOptions,
destinationPath: filename
};
download_1.downloadInExternalProcess(opts).then(() => {
fs.stat(filename, (err, stat) => {
if (err) {
result.emit('error', err);
return;
}
fs.readFile(filename, (err, data) => {
if (err) {
result.emit('error', err);
return;
}
fs.unlink(filename, () => {
result.emit('data', new VinylFile({
path: path.normalize(requestOptions.path),
stat: stat,
base: path.normalize(requestOptions.path),
contents: data
}));
result.emit('end');
});
});
});
}, (err) => {
result.emit('error', err);
});
return result;
}
exports.download = download;
build/lib/util.ts
+3 -38
View File
@@ -17,9 +17,6 @@ import * as git from './git';
import * as VinylFile from 'vinyl'; import * as VinylFile from 'vinyl';
import { ThroughStream } from 'through'; import { ThroughStream } from 'through';
import * as sm from 'source-map'; import * as sm from 'source-map';
import { IDownloadOptions, downloadInExternalProcess, IDownloadRequestOptions } from '../download/download';
const REPO_ROOT = path.join(__dirname, '../../');
export interface ICancellationToken { export interface ICancellationToken {
isCancellationRequested(): boolean; isCancellationRequested(): boolean;
@@ -96,6 +93,9 @@ export function fixWin32DirectoryPermissions(): NodeJS.ReadWriteStream {
export function setExecutableBit(pattern?: string | string[]): NodeJS.ReadWriteStream { export function setExecutableBit(pattern?: string | string[]): NodeJS.ReadWriteStream {
const setBit = es.mapSync<VinylFile, VinylFile>(f => { const setBit = es.mapSync<VinylFile, VinylFile>(f => {
if (!f.stat) {
f.stat = { isFile() { return true; } } as any;
}
f.stat.mode = /* 100755 */ 33261; f.stat.mode = /* 100755 */ 33261;
return f; return f;
}); });
@@ -281,38 +281,3 @@ export function versionStringToNumber(versionStr: string) {
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10); return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
} }
export function download(requestOptions: IDownloadRequestOptions): NodeJS.ReadWriteStream {
const result = es.through();
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
const opts: IDownloadOptions = {
requestOptions: requestOptions,
destinationPath: filename
};
downloadInExternalProcess(opts).then(() => {
fs.stat(filename, (err, stat) => {
if (err) {
result.emit('error', err);
return;
}
fs.readFile(filename, (err, data) => {
if (err) {
result.emit('error', err);
return;
}
fs.unlink(filename, () => {
result.emit('data', new VinylFile({
path: path.normalize(requestOptions.path),
stat: stat,
base: path.normalize(requestOptions.path),
contents: data
}));
result.emit('end');
});
});
});
}, (err) => {
result.emit('error', err);
});
return result;
}