From f23ed65688e94c807412c6e86230e6e78405bd4f Mon Sep 17 00:00:00 2001 From: Alexandru Dima Date: Wed, 23 Dec 2020 13:21:55 +0100 Subject: [PATCH] Compute node modules cache key in JS to avoid globbing --- .github/workflows/ci.yml | 436 +----------------- .../common/computeNodeModulesCacheKey.js | 25 + .../common/computeNodeModulesCacheKey.ts | 32 ++ .../darwin/continuous-build-darwin.yml | 9 +- .../darwin/product-build-darwin.yml | 8 +- .../linux/continuous-build-linux.yml | 9 +- .../linux/product-build-alpine.yml | 8 +- .../linux/product-build-linux.yml | 8 +- build/azure-pipelines/product-compile.yml | 7 +- build/azure-pipelines/product-hygiene.yml | 5 +- .../azure-pipelines/web/product-build-web.yml | 10 +- .../win32/continuous-build-win32.yml | 9 +- .../win32/product-build-win32.yml | 8 +- 13 files changed, 128 insertions(+), 446 deletions(-) create mode 100644 build/azure-pipelines/common/computeNodeModulesCacheKey.js create mode 100644 build/azure-pipelines/common/computeNodeModulesCacheKey.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60695dc8aa5..3b1b725e48f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,12 +23,15 @@ jobs: with: node-version: 12 + - name: Compute node modules cache key + id: nodeModulesCacheKey + run: echo "::set-output name=value::$(node build/azure-pipelines/common/computeNodeModulesCacheKey.js)" - name: Cache node modules id: cacheNodeModules uses: actions/cache@v2 with: path: "**/node_modules" - key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-cacheNodeModules7-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-cacheNodeModules7- - name: Get yarn cache directory path id: yarnCacheDirPath @@ -39,7 +42,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarnCacheDirPath.outputs.dir }} - key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-yarnCacheDir-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-yarnCacheDir- - name: Execute yarn if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} @@ -54,409 +57,6 @@ jobs: - name: Run Valid Layers Checks run: yarn valid-layers-check - # build-compile-core: - # name: "Build: Compile Core" - # runs-on: ubuntu-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Cache node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - # - name: Get yarn cache directory path - # id: yarnCacheDirPath - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # run: echo "::set-output name=dir::$(yarn cache dir)" - # - name: Cache yarn directory - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # uses: actions/cache@v2 - # with: - # path: ${{ steps.yarnCacheDirPath.outputs.dir }} - # key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-yarnCacheDir- - # - name: Execute yarn - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # env: - # PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - # ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - # run: yarn --frozen-lockfile --network-timeout 180000 - - # - name: Cache compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Compile Core - # if: ${{ steps.cacheCompiledCoreCode.outputs.cache-hit != 'true' }} - # run: yarn gulp compile-build - - # - name: Minify VS Code - # if: ${{ steps.cacheCompiledCoreCode.outputs.cache-hit != 'true' }} - # run: yarn gulp minify-vscode - - # build-compile-extensions: - # name: "Build: Compile Extensions" - # runs-on: ubuntu-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Cache node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - # - name: Get yarn cache directory path - # id: yarnCacheDirPath - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # run: echo "::set-output name=dir::$(yarn cache dir)" - # - name: Cache yarn directory - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # uses: actions/cache@v2 - # with: - # path: ${{ steps.yarnCacheDirPath.outputs.dir }} - # key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-yarnCacheDir- - # - name: Execute yarn - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # env: - # PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - # ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - # run: yarn --frozen-lockfile --network-timeout 180000 - - # - name: Cache compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Compile Extensions - # if: ${{ steps.cacheCompiledExtensionsCode.outputs.cache-hit != 'true' }} - # run: yarn gulp compile-extensions-build - - # build-linux-unit-tests: - # name: "Build: Linux Unit Tests" - # needs: [build-compile-core, build-compile-extensions] - # runs-on: ubuntu-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # # TODO: rename azure-pipelines/linux/xvfb.init to github-actions - # - name: Setup Build Environment - # run: | - # sudo apt-get update - # sudo apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 libgbm1 - # sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb - # sudo chmod +x /etc/init.d/xvfb - # sudo update-rc.d xvfb defaults - # sudo service xvfb start - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Restore cached node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - - # - name: Restore compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Restore compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Build VS Code - # run: yarn gulp vscode-linux-x64-min-ci - - # - name: Download Electron - # run: yarn electron x64 - - # - name: Run Unit Tests (Electron) - # run: DISPLAY=:10 ./scripts/test.sh --build - - # - name: Download Playwright - # run: node ./node_modules/playwright/install.js - - # - name: Run Unit Tests (Browser) - # run: DISPLAY=:10 yarn test-browser --build --browser chromium - - # build-linux-integration-tests: - # name: "Build: Linux Integration Tests" - # needs: [build-compile-core, build-compile-extensions] - # runs-on: ubuntu-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # # TODO: rename azure-pipelines/linux/xvfb.init to github-actions - # - name: Setup Build Environment - # run: | - # sudo apt-get update - # sudo apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 libgbm1 - # sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb - # sudo chmod +x /etc/init.d/xvfb - # sudo update-rc.d xvfb defaults - # sudo service xvfb start - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Restore cached node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - - # - name: Restore compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Restore compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Build VS Code - # run: yarn gulp vscode-linux-x64-min-ci - - # - name: Download Electron - # run: yarn electron x64 - - # - name: Run Integration Tests (Electron) - # run: DISPLAY=:10 ./scripts/test-integration.sh --build - - # build-darwin-node-modules: - # name: "Build: macOS Node Modules" - # runs-on: macos-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Cache node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - # - name: Get yarn cache directory path - # id: yarnCacheDirPath - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # run: echo "::set-output name=dir::$(yarn cache dir)" - # - name: Cache yarn directory - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # uses: actions/cache@v2 - # with: - # path: ${{ steps.yarnCacheDirPath.outputs.dir }} - # key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-yarnCacheDir- - # - name: Execute yarn - # if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} - # env: - # PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - # ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - # run: yarn --frozen-lockfile --network-timeout 180000 - - # build-darwin-unit-tests: - # name: "Build: macOS Unit Tests" - # needs: [build-compile-core, build-compile-extensions, build-darwin-node-modules] - # runs-on: macos-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Restore cached node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - - # - name: Restore compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Restore compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Build VS Code - # run: yarn gulp vscode-darwin-x64-min-ci - - # - name: Download Electron - # run: yarn electron x64 - - # - name: Run Unit Tests (Electron) - # run: ./scripts/test.sh --build - - # - name: Download Playwright - # run: node ./node_modules/playwright/install.js - - # - name: Run Unit Tests (Browser) - # run: yarn test-browser --build --browser chromium --browser webkit --browser firefox - - # build-darwin-integration-tests: - # name: "Build: macOS Integration Tests" - # needs: [build-compile-core, build-compile-extensions, build-darwin-node-modules] - # runs-on: macos-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Restore cached node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - - # - name: Restore compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Restore compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Build VS Code - # run: yarn gulp vscode-darwin-x64-min-ci - - # - name: Download Electron - # run: yarn electron x64 - - # - name: Run Integration Tests (Electron) - # run: ./scripts/test-integration.sh --build - - # build-darwin-smoke-tests: - # name: "Build: macOS Smoke Tests" - # needs: [build-compile-core, build-compile-extensions, build-darwin-node-modules] - # runs-on: macos-latest - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # steps: - # - uses: actions/checkout@v2 - - # - uses: actions/setup-node@v2-beta - # with: - # node-version: 12 - - # - name: Restore cached node modules - # id: cacheNodeModules - # uses: actions/cache@v2 - # with: - # path: '**/node_modules' - # key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} - # restore-keys: ${{ runner.os }}-cacheNodeModules7- - - # - name: Restore compiled core code - # id: cacheCompiledCoreCode - # uses: actions/cache@v2 - # with: - # path: | - # out-build - # out-vscode-min - # key: cacheCompiledCoreCode-${{ github.sha }} - - # - name: Restore compiled extensions code - # id: cacheCompiledExtensionsCode - # uses: actions/cache@v2 - # with: - # path: .build - # key: cacheCompiledExtensionsCode-${{ github.sha }} - - # - name: Build VS Code - # run: yarn gulp vscode-darwin-x64-min-ci - - # - name: Run Smoke Tests (Electron) - # continue-on-error: true - # run: | - # set -e - # APP_ROOT=$(GITHUB_WORKSPACE)/VSCode-darwin-x64 - # APP_NAME="`ls $APP_ROOT | head -n 1`" - # yarn smoketest --build "$APP_ROOT/$APP_NAME" - linux: name: Linux runs-on: ubuntu-latest @@ -479,12 +79,15 @@ jobs: with: node-version: 12 + - name: Compute node modules cache key + id: nodeModulesCacheKey + run: echo "::set-output name=value::$(node build/azure-pipelines/common/computeNodeModulesCacheKey.js)" - name: Cache node modules id: cacheNodeModules uses: actions/cache@v2 with: path: "**/node_modules" - key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-cacheNodeModules7-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-cacheNodeModules7- - name: Get yarn cache directory path id: yarnCacheDirPath @@ -495,7 +98,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarnCacheDirPath.outputs.dir }} - key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-yarnCacheDir-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-yarnCacheDir- - name: Execute yarn if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} @@ -536,6 +139,9 @@ jobs: with: python-version: "2.x" + - name: Compute node modules cache key + id: nodeModulesCacheKey + run: echo "::set-output name=value::$(node build/azure-pipelines/common/computeNodeModulesCacheKey.js)" - name: Cache node modules id: cacheNodeModules uses: actions/cache@v2 @@ -546,7 +152,7 @@ jobs: !**/Release/**/*.ilk !**/Release/**/*.obj !**/Release/**/*.tlog - key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-cacheNodeModules7-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-cacheNodeModules7- - name: Get yarn cache directory path id: yarnCacheDirPath @@ -557,7 +163,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarnCacheDirPath.outputs.dir }} - key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-yarnCacheDir-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-yarnCacheDir- - name: Execute yarn if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} @@ -590,12 +196,15 @@ jobs: with: node-version: 12 + - name: Compute node modules cache key + id: nodeModulesCacheKey + run: echo "::set-output name=value::$(node build/azure-pipelines/common/computeNodeModulesCacheKey.js)" - name: Cache node modules id: cacheNodeModules uses: actions/cache@v2 with: path: "**/node_modules" - key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-cacheNodeModules7-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-cacheNodeModules7- - name: Get yarn cache directory path id: yarnCacheDirPath @@ -606,7 +215,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarnCacheDirPath.outputs.dir }} - key: ${{ runner.os }}-yarnCacheDir-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-yarnCacheDir-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-yarnCacheDir- - name: Execute yarn if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} @@ -639,12 +248,15 @@ jobs: with: node-version: 12 + - name: Compute node modules cache key + id: nodeModulesCacheKey + run: echo "::set-output name=value::$(node build/azure-pipelines/common/computeNodeModulesCacheKey.js)" - name: Cache node modules id: cacheNodeModules uses: actions/cache@v2 with: path: "**/node_modules" - key: ${{ runner.os }}-cacheNodeModules7-${{ hashFiles('.yarnrc', 'remote/.yarnrc', '**/yarn.lock', '!**/node_modules/**/yarn.lock', '!**/.*/**/yarn.lock') }} + key: ${{ runner.os }}-cacheNodeModules7-${{ steps.nodeModulesCacheKey.outputs.value }} restore-keys: ${{ runner.os }}-cacheNodeModules7- - name: Execute yarn if: ${{ steps.cacheNodeModules.outputs.cache-hit != 'true' }} diff --git a/build/azure-pipelines/common/computeNodeModulesCacheKey.js b/build/azure-pipelines/common/computeNodeModulesCacheKey.js new file mode 100644 index 00000000000..b471d5c84cc --- /dev/null +++ b/build/azure-pipelines/common/computeNodeModulesCacheKey.js @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); +const { dirs } = require('../../npm/dirs'); +const ROOT = path.join(__dirname, '../../../'); +const shasum = crypto.createHash('sha1'); +shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt'))); +shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc'))); +shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc'))); +// Add `yarn.lock` files +for (let dir of dirs) { + const yarnLockPath = path.join(ROOT, dir, 'yarn.lock'); + shasum.update(fs.readFileSync(yarnLockPath)); +} +// Add any other command line arguments +for (let i = 2; i < process.argv.length; i++) { + shasum.update(process.argv[i]); +} +process.stdout.write(shasum.digest('hex')); diff --git a/build/azure-pipelines/common/computeNodeModulesCacheKey.ts b/build/azure-pipelines/common/computeNodeModulesCacheKey.ts new file mode 100644 index 00000000000..638f115a7fb --- /dev/null +++ b/build/azure-pipelines/common/computeNodeModulesCacheKey.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as fs from 'fs'; +import * as path from 'path'; +import * as crypto from 'crypto'; +const { dirs } = require('../../npm/dirs'); + +const ROOT = path.join(__dirname, '../../../'); + +const shasum = crypto.createHash('sha1'); + +shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt'))); +shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc'))); +shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc'))); + +// Add `yarn.lock` files +for (let dir of dirs) { + const yarnLockPath = path.join(ROOT, dir, 'yarn.lock'); + shasum.update(fs.readFileSync(yarnLockPath)); +} + +// Add any other command line arguments +for (let i = 2; i < process.argv.length; i++) { + shasum.update(process.argv[i]); +} + +process.stdout.write(shasum.digest('hex')); diff --git a/build/azure-pipelines/darwin/continuous-build-darwin.yml b/build/azure-pipelines/darwin/continuous-build-darwin.yml index 75c28106c5e..75bd292fe42 100644 --- a/build/azure-pipelines/darwin/continuous-build-darwin.yml +++ b/build/azure-pipelines/darwin/continuous-build-darwin.yml @@ -7,9 +7,14 @@ steps: inputs: versionSpec: "1.x" + - script: | + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache flags + - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" @@ -20,7 +25,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index ac98b5bbf3a..fc4f8741cd0 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -54,13 +54,13 @@ steps: condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - script: | - echo -n $(VSCODE_ARCH) > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -92,7 +92,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/linux/continuous-build-linux.yml b/build/azure-pipelines/linux/continuous-build-linux.yml index 44cb9829472..62712e07336 100644 --- a/build/azure-pipelines/linux/continuous-build-linux.yml +++ b/build/azure-pipelines/linux/continuous-build-linux.yml @@ -16,9 +16,14 @@ steps: inputs: versionSpec: "1.x" + - script: | + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache flags + - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" @@ -29,7 +34,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/linux/product-build-alpine.yml b/build/azure-pipelines/linux/product-build-alpine.yml index c04220403f8..8ebcd313090 100644 --- a/build/azure-pipelines/linux/product-build-alpine.yml +++ b/build/azure-pipelines/linux/product-build-alpine.yml @@ -57,13 +57,13 @@ steps: condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - script: | - echo -n "alpine" > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js "alpine" $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -85,7 +85,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml index 81e4aba30e4..558d6573f1e 100644 --- a/build/azure-pipelines/linux/product-build-linux.yml +++ b/build/azure-pipelines/linux/product-build-linux.yml @@ -48,13 +48,13 @@ steps: condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - script: | - echo -n $(VSCODE_ARCH) > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -96,7 +96,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index 0be80afa471..efbe6dfd60d 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -38,13 +38,12 @@ steps: - script: | mkdir -p .build - echo -n $(VSCODE_ARCH) > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin + node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -79,7 +78,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/product-hygiene.yml b/build/azure-pipelines/product-hygiene.yml index 0ee16824cde..b776e277abe 100644 --- a/build/azure-pipelines/product-hygiene.yml +++ b/build/azure-pipelines/product-hygiene.yml @@ -38,13 +38,12 @@ steps: - script: | mkdir -p .build - echo -n $(VSCODE_ARCH) > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin + node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" diff --git a/build/azure-pipelines/web/product-build-web.yml b/build/azure-pipelines/web/product-build-web.yml index 4ffb93f4571..e7699fd52c5 100644 --- a/build/azure-pipelines/web/product-build-web.yml +++ b/build/azure-pipelines/web/product-build-web.yml @@ -48,13 +48,13 @@ steps: condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - script: | - echo -n "web" > .build/arch - echo -n $ENABLE_TERRAPIN > .build/terrapin - displayName: Prepare yarn cache flag + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js "web" $ENABLE_TERRAPIN > .build/yarnlockhash + displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -76,7 +76,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/win32/continuous-build-win32.yml b/build/azure-pipelines/win32/continuous-build-win32.yml index 7bd18707381..61b99ce93a7 100644 --- a/build/azure-pipelines/win32/continuous-build-win32.yml +++ b/build/azure-pipelines/win32/continuous-build-win32.yml @@ -12,9 +12,14 @@ steps: versionSpec: "2.x" addToPath: true + - powershell: | + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache flags + - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" @@ -27,7 +32,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "vscode-build-cache" condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index 04800ad63b1..772efb7f5ff 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -52,13 +52,13 @@ steps: condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - powershell: | - "$(VSCODE_ARCH)" | Out-File -Encoding ascii -NoNewLine .build\arch - "$env:ENABLE_TERRAPIN" | Out-File -Encoding ascii -NoNewLine .build\terrapin + mkdir -p .build + node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" @@ -77,7 +77,7 @@ steps: - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 inputs: - keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock" + keyfile: ".build/yarnlockhash" targetfolder: "**/node_modules, !**/node_modules/**/node_modules" vstsFeed: "npm-vscode" condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))