feat: switch to npm as default package manager (#226927)

* feat: move from yarn to npm

* chore: skip yarn.lock files

* fix: playwright download

* chore: fix compile and hygiene

* chore: bump vsce@2.17.0

Refs https://github.com/microsoft/vscode-vsce/commit/8b49e9dfdf909ad3af2b9ec9c825f5b501f6d75e

* test: update results for bat and sh colorizer tests

* fix: add missing lock files for windows

* fix: switch to legacy-peer-deps

* chore: update markdown-it@14.1.0

Refs https://github.com/markdown-it/markdown-it/commit/737c95a12976357df99652e4b51d831cac4a75aa

esbuild step in extensions-ci-pr was previously using markdown-it
from root which had userland punycode and was able to compile successfully.

* ci: increase pr timeout for windows integration tests

* chore: fix product build

* build: ignore extension dev dependency for rcedit

* build: fix working directory inside container

* build: fix dependency generation

* npm: update dependencies

* ci: use global npmrc

* ci: update cache

* ci: setup global npmrc for private npm auth

* build: fix extension bundling

* chore: sync npm dependencies

* ci: debug env variables for container

* ci: fix win32 cli pipeline

* build: fix npmrc config usage for build/ and remote/ dirs

* fix: windows build

* fix: container builds

* fix: markdown-language-features tests and bundling

```
[03:58:22] Error: Command failed: /Users/demohan/.nvm/versions/node/v20.15.1/bin/node /Users/demohan/github/vscode/extensions/markdown-language-features/esbuild-notebook.js --outputRoot /Users/demohan/github/vscode/.build/extensions/markdown-language-features
✘ [ERROR] Could not resolve "punycode"

    extensions/markdown-language-features/node_modules/markdown-it/lib/index.js:14:27:
      14 │ var punycode     = require('punycode');
         ╵                            ~~~~~~~~~~

  The package "punycode" wasn't found on the file system but is built into node. Are you trying to bundle for node? You can use "platform: 'node'" to do that, which will remove this error.
```

Adds userland package based on https://github.com/markdown-it/markdown-it/commit/beed9aee2c1b44819afc25d485e86a6c699b6ef0

* fix: container builds for distro

* chore: update yarn occurrences

* fixup! chore: bump vsce@2.17.0

Uses the closest version to `main` branch that does not
include https://github.com/microsoft/vscode-vsce/commit/d3cc84cdec99b8e9f24be63d1a2a54abe908f68b
while still having the fix https://github.com/microsoft/vscode-vsce/commit/8b49e9dfdf909ad3af2b9ec9c825f5b501f6d75e

* chore: sync npm dependencies

* chore: sync npm dependencies

* chore: sync npm dependencies

* chore: throw error when yarn is used for installation

* chore: add review feedback

* chore: switch exec => run where needed

* chore: npm sync dependencies

* fix: markdown-language-features bundling

```
✘ [ERROR] Could not resolve "punycode"

    extensions/markdown-language-features/node_modules/markdown-it/lib/index.js:14:27:
      14 │ var punycode     = require('punycode');
         ╵                            ~~~~~~~~~~

  The package "punycode" wasn't found on the file system but is built into node. Are you trying to bundle for node? You can use "platform: 'node'" to do that, which will remove this error.
```

Adds missing userland package based on markdown-it/markdown-it@beed9ae,
can be removed once we update markdown-it >= 14.1.0

* ci: rename no-yarn-lock-changes.yml

* chore: sync npm dependencies

* ci: restore no-yarn-lock-changes.yml

We can disable it in a separate PR to keep the required
checks happy and also need workflow edit perms.

* chore: sync npm dependencies

* ci: rebuild cache

* ci: fix no-package-lock-changes.yml

* chore: bump distro

* chore: rm yarn.lock files

* chore: rm yarn.lock files without dependencies

* chore: add vscode-selfhost-import-aid to postinstall dirs

* chore: bump distro
This commit is contained in:
Robo
2024-09-06 22:18:02 +09:00
committed by GitHub
parent 5ae8ffb7e3
commit b5a6aa14a8
306 changed files with 38279 additions and 23070 deletions
+12 -12
View File
@@ -40,24 +40,23 @@ steps:
displayName: Extract openssl prebuilt
- ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}:
- script: node build/setup-npm-registry.js $NPM_REGISTRY
- script: node build/setup-npm-registry.js $NPM_REGISTRY build
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM Registry
- script: |
set -e
npm config set registry "$NPM_REGISTRY" --location=project
# npm >v7 deprecated the `always-auth` config option, refs npm/cli@72a7eeb
# following is a workaround for yarn to send authorization header
# for GET requests to the registry.
echo "always-auth=true" >> .npmrc
yarn config set registry "$NPM_REGISTRY"
# Set the private NPM registry to the global npmrc file
# so that authentication works for subfolders like build/, remote/, extensions/ etc
# which does not have their own .npmrc file
npm config set registry "$NPM_REGISTRY"
echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)"
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM & Yarn
displayName: Setup NPM
- task: npmAuthenticate@0
inputs:
workingFile: .npmrc
workingFile: $(NPMRC_PATH)
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM Authentication
@@ -65,13 +64,14 @@ steps:
set -e
for i in {1..5}; do # try 5 times
yarn --cwd build --frozen-lockfile --check-files && break
npm ci && break
if [ $i -eq 5 ]; then
echo "Yarn failed too many times" >&2
echo "Npm install failed too many times" >&2
exit 1
fi
echo "Yarn failed $i, trying again..."
echo "Npm install failed $i, trying again..."
done
workingDirectory: build
displayName: Install build dependencies
- script: |
@@ -62,18 +62,17 @@ steps:
- script: |
set -e
npm config set registry "$NPM_REGISTRY" --location=project
# npm >v7 deprecated the `always-auth` config option, refs npm/cli@72a7eeb
# following is a workaround for yarn to send authorization header
# for GET requests to the registry.
echo "always-auth=true" >> .npmrc
yarn config set registry "$NPM_REGISTRY"
# Set the private NPM registry to the global npmrc file
# so that authentication works for subfolders like build/, remote/, extensions/ etc
# which does not have their own .npmrc file
npm config set registry "$NPM_REGISTRY"
echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)"
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM & Yarn
displayName: Setup NPM
- task: npmAuthenticate@0
inputs:
workingFile: .npmrc
workingFile: $(NPMRC_PATH)
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM Authentication
@@ -89,28 +88,31 @@ steps:
- script: |
set -e
# To workaround the issue of yarn not respecting the registry value from .npmrc
yarn config set registry "$NPM_REGISTRY"
for i in {1..5}; do # try 5 times
yarn --cwd build --frozen-lockfile --check-files && break
npm ci && break
if [ $i -eq 5 ]; then
echo "Yarn failed too many times" >&2
echo "Npm install failed too many times" >&2
exit 1
fi
echo "Yarn failed $i, trying again..."
echo "Npm install failed $i, trying again..."
done
workingDirectory: build
displayName: Install build dependencies
- script: |
set -e
export VSCODE_SYSROOT_PREFIX='-glibc-2.17'
source ./build/azure-pipelines/linux/setup-env.sh --only-remote
for i in {1..5}; do # try 5 times
yarn --frozen-lockfile --check-files && break
npm ci && break
if [ $i -eq 5 ]; then
echo "Yarn failed too many times" >&2
echo "Npm install failed too many times" >&2
exit 1
fi
echo "Yarn failed $i, trying again..."
echo "Npm install failed $i, trying again..."
done
env:
npm_config_arch: $(NPM_ARCH)
@@ -134,7 +136,7 @@ steps:
- script: |
set -e
yarn gulp vscode-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-linux-$(VSCODE_ARCH)-min-ci
ARCHIVE_PATH=".build/linux/client/code-${{ parameters.VSCODE_QUALITY }}-$(VSCODE_ARCH)-$(date +%s).tar.gz"
mkdir -p $(dirname $ARCHIVE_PATH)
echo "##vso[task.setvariable variable=CLIENT_PATH]$ARCHIVE_PATH"
@@ -152,7 +154,7 @@ steps:
- script: |
set -e
export VSCODE_NODE_GLIBC="-glibc-2.17"
yarn gulp vscode-reh-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-reh-linux-$(VSCODE_ARCH)-min-ci
mv ../vscode-reh-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH) # TODO@joaomoreno
ARCHIVE_PATH=".build/linux/server/vscode-server-linux-legacy-$(VSCODE_ARCH).tar.gz"
UNARCHIVE_PATH="`pwd`/../vscode-server-linux-$(VSCODE_ARCH)"
@@ -167,7 +169,7 @@ steps:
- script: |
set -e
export VSCODE_NODE_GLIBC="-glibc-2.17"
yarn gulp vscode-reh-web-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-reh-web-linux-$(VSCODE_ARCH)-min-ci
mv ../vscode-reh-web-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH)-web # TODO@joaomoreno
ARCHIVE_PATH=".build/linux/web/vscode-server-linux-legacy-$(VSCODE_ARCH)-web.tar.gz"
mkdir -p $(dirname $ARCHIVE_PATH)
@@ -15,7 +15,7 @@ parameters:
default: false
steps:
- script: yarn npm-run-all -lp "electron $(VSCODE_ARCH)" "playwright-install"
- script: npm exec -- npm-run-all -lp "electron $(VSCODE_ARCH)" "playwright-install"
env:
GITHUB_TOKEN: "$(github-distro-mixin-password)"
displayName: Download Electron and Playwright
@@ -42,10 +42,10 @@ steps:
DISPLAY: ":10"
displayName: Run unit tests (Electron) [AMD]
timeoutInMinutes: 15
- script: yarn test-node-amd
- script: npm run test-node-amd
displayName: Run unit tests (node.js) [AMD]
timeoutInMinutes: 15
- script: yarn test-browser-amd-no-install --browser chromium --tfs "Browser Unit Tests"
- script: npm run test-browser-amd-no-install -- --browser chromium --tfs "Browser Unit Tests"
env:
DEBUG: "*browser*"
displayName: Run unit tests (Browser, Chromium) [AMD]
@@ -56,10 +56,10 @@ steps:
DISPLAY: ":10"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
- script: yarn test-node
- script: npm run test-node
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
- script: yarn test-browser-no-install --browser chromium --tfs "Browser Unit Tests"
- script: npm run test-browser-no-install -- --browser chromium --tfs "Browser Unit Tests"
env:
DEBUG: "*browser*"
displayName: Run unit tests (Browser, Chromium)
@@ -71,10 +71,10 @@ steps:
- script: ./scripts/test-amd.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron) [AMD]
timeoutInMinutes: 15
- script: yarn test-node-amd --build
- script: npm run test-node-amd -- --build
displayName: Run unit tests (node.js) [AMD]
timeoutInMinutes: 15
- script: yarn test-browser-amd-no-install --build --browser chromium --tfs "Browser Unit Tests"
- script: npm run test-browser-amd-no-install -- --build --browser chromium --tfs "Browser Unit Tests"
env:
DEBUG: "*browser*"
displayName: Run unit tests (Browser, Chromium) [AMD]
@@ -83,10 +83,10 @@ steps:
- script: ./scripts/test.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
- script: yarn test-node --build
- script: npm run test-node -- --build
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
- script: yarn test-browser-no-install --build --browser chromium --tfs "Browser Unit Tests"
- script: npm run test-browser-no-install -- --build --browser chromium --tfs "Browser Unit Tests"
env:
DEBUG: "*browser*"
displayName: Run unit tests (Browser, Chromium)
@@ -95,7 +95,7 @@ steps:
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
yarn gulp \
npm run gulp \
compile-extension:configuration-editing \
compile-extension:css-language-features-server \
compile-extension:emmet \
@@ -198,34 +198,35 @@ steps:
condition: succeededOrFailed()
- ${{ if eq(parameters.VSCODE_QUALITY, 'oss') }}:
- script: yarn --cwd test/smoke compile
- script: npm run compile
workingDirectory: test/smoke
displayName: Compile smoke tests
- script: yarn gulp compile-extension:markdown-language-features compile-extension:ipynb compile-extension-media compile-extension:vscode-test-resolver
- script: npm run gulp compile-extension:markdown-language-features compile-extension:ipynb compile-extension-media compile-extension:vscode-test-resolver
displayName: Build extensions for smoke tests
- script: yarn gulp node
- script: npm run gulp node
displayName: Download node.js for remote smoke tests
retryCountOnTaskFailure: 3
- script: yarn smoketest-no-compile --tracing
- script: npm run smoketest-no-compile -- --tracing
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
- script: yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
- script: npm run smoketest-no-compile -- --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
timeoutInMinutes: 20
displayName: Run smoke tests (Browser, Chromium)
- script: yarn smoketest-no-compile --remote --tracing
- script: npm run smoketest-no-compile -- --remote --tracing
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
- ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}:
- script: yarn smoketest-no-compile --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)"
- script: npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)"
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
- script: yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
- script: npm run smoketest-no-compile -- --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
env:
VSCODE_REMOTE_SERVER_PATH: $(agent.builddirectory)/vscode-server-linux-$(VSCODE_ARCH)-web
timeoutInMinutes: 20
@@ -233,10 +234,10 @@ steps:
- script: |
set -e
yarn gulp compile-extension:vscode-test-resolver
npm run gulp compile-extension:vscode-test-resolver
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-server-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
npm run smoketest-no-compile -- --tracing --remote --build "$APP_PATH"
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
@@ -72,12 +72,12 @@ steps:
condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM Registry
- script: mkdir -p .build && node build/azure-pipelines/common/computeNodeModulesCacheKey.js linux $VSCODE_ARCH > .build/yarnlockhash
- script: mkdir -p .build && node build/azure-pipelines/common/computeNodeModulesCacheKey.js linux $VSCODE_ARCH > .build/packagelockhash
displayName: Prepare node_modules cache key
- task: Cache@2
inputs:
key: '"node_modules" | .build/yarnlockhash'
key: '"node_modules" | .build/packagelockhash'
path: .build/node_modules_cache
cacheHitVar: NODE_MODULES_RESTORED
displayName: Restore node_modules cache
@@ -88,18 +88,17 @@ steps:
- script: |
set -e
npm config set registry "$NPM_REGISTRY" --location=project
# npm >v7 deprecated the `always-auth` config option, refs npm/cli@72a7eeb
# following is a workaround for yarn to send authorization header
# for GET requests to the registry.
echo "always-auth=true" >> .npmrc
yarn config set registry "$NPM_REGISTRY"
# Set the private NPM registry to the global npmrc file
# so that authentication works for subfolders like build/, remote/, extensions/ etc
# which does not have their own .npmrc file
npm config set registry "$NPM_REGISTRY"
echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)"
condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM & Yarn
displayName: Setup NPM
- task: npmAuthenticate@0
inputs:
workingFile: .npmrc
workingFile: $(NPMRC_PATH)
condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none'))
displayName: Setup NPM Authentication
@@ -107,13 +106,14 @@ steps:
set -e
for i in {1..5}; do # try 5 times
yarn --cwd build --frozen-lockfile --check-files && break
npm ci && break
if [ $i -eq 5 ]; then
echo "Yarn failed too many times" >&2
echo "Npm install failed too many times" >&2
exit 1
fi
echo "Yarn failed $i, trying again..."
echo "Npm install failed $i, trying again..."
done
workingDirectory: build
displayName: Install build dependencies
condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
@@ -138,12 +138,12 @@ steps:
source ./build/azure-pipelines/linux/setup-env.sh
for i in {1..5}; do # try 5 times
yarn --frozen-lockfile --check-files && break
npm ci && break
if [ $i -eq 5 ]; then
echo "Yarn failed too many times" >&2
echo "Npm install failed too many times" >&2
exit 1
fi
echo "Yarn failed $i, trying again..."
echo "Npm install failed $i, trying again..."
done
env:
npm_config_arch: $(NPM_ARCH)
@@ -190,7 +190,7 @@ steps:
- ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}:
- script: |
set -e
yarn gulp vscode-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-linux-$(VSCODE_ARCH)-min-ci
ARCHIVE_PATH=".build/linux/client/code-${{ parameters.VSCODE_QUALITY }}-$(VSCODE_ARCH)-$(date +%s).tar.gz"
mkdir -p $(dirname $ARCHIVE_PATH)
echo "##vso[task.setvariable variable=CLIENT_PATH]$ARCHIVE_PATH"
@@ -223,7 +223,7 @@ steps:
- script: |
set -e
yarn gulp vscode-reh-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-reh-linux-$(VSCODE_ARCH)-min-ci
mv ../vscode-reh-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH) # TODO@joaomoreno
ARCHIVE_PATH=".build/linux/server/vscode-server-linux-$(VSCODE_ARCH).tar.gz"
UNARCHIVE_PATH="`pwd`/../vscode-server-linux-$(VSCODE_ARCH)"
@@ -237,7 +237,7 @@ steps:
- script: |
set -e
yarn gulp vscode-reh-web-linux-$(VSCODE_ARCH)-min-ci
npm run gulp vscode-reh-web-linux-$(VSCODE_ARCH)-min-ci
mv ../vscode-reh-web-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH)-web # TODO@joaomoreno
ARCHIVE_PATH=".build/linux/web/vscode-server-linux-$(VSCODE_ARCH)-web.tar.gz"
mkdir -p $(dirname $ARCHIVE_PATH)
@@ -278,7 +278,7 @@ steps:
displayName: Check GLIBC and GLIBCXX dependencies in server archive
- ${{ else }}:
- script: yarn gulp "transpile-client-swc" "transpile-extensions"
- script: npm run gulp "transpile-client-swc" "transpile-extensions"
env:
GITHUB_TOKEN: "$(github-distro-mixin-password)"
displayName: Transpile client and extensions
@@ -297,14 +297,14 @@ steps:
- ${{ if and(ne(parameters.VSCODE_CIBUILD, true), ne(parameters.VSCODE_QUALITY, 'oss')) }}:
- script: |
set -e
yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-deb"
npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-deb"
env:
GITHUB_TOKEN: "$(github-distro-mixin-password)"
displayName: Prepare deb package
- script: |
set -e
yarn gulp "vscode-linux-$(VSCODE_ARCH)-build-deb"
npm run gulp "vscode-linux-$(VSCODE_ARCH)-build-deb"
echo "##vso[task.setvariable variable=DEB_PATH]$(ls .build/linux/deb/*/deb/*.deb)"
displayName: Build deb package
@@ -320,20 +320,20 @@ steps:
fi
export VSCODE_SYSROOT_DIR=$(Build.SourcesDirectory)/.build/sysroots
export STRIP="$VSCODE_SYSROOT_DIR/$TRIPLE/$TRIPLE/bin/strip"
yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-rpm"
npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-rpm"
env:
VSCODE_ARCH: $(VSCODE_ARCH)
displayName: Prepare rpm package
- script: |
set -e
yarn gulp "vscode-linux-$(VSCODE_ARCH)-build-rpm"
npm run gulp "vscode-linux-$(VSCODE_ARCH)-build-rpm"
echo "##vso[task.setvariable variable=RPM_PATH]$(ls .build/linux/rpm/*/*.rpm)"
displayName: Build rpm package
- script: |
set -e
yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-snap"
npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-snap"
ARCHIVE_PATH=".build/linux/snap-tarball/snap-$(VSCODE_ARCH).tar.gz"
mkdir -p $(dirname $ARCHIVE_PATH)
tar -czf $ARCHIVE_PATH -C .build/linux snap
@@ -22,17 +22,11 @@ steps:
sudo apt-get upgrade -y
sudo apt-get install -y curl apt-transport-https ca-certificates
# Yarn
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
sudo apt-get update
sudo apt-get install -y yarn
# Define variables
SNAP_ROOT="$(pwd)/.build/linux/snap/$(VSCODE_ARCH)"
# Install build dependencies
(cd build && yarn)
(cd build && npm ci)
# Unpack snap tarball artifact, in order to preserve file perms
(cd .build/linux && tar -xzf snap-tarball/snap-$(VSCODE_ARCH).tar.gz)
@@ -10,7 +10,7 @@ elif [ "$VSCODE_ARCH" == "armhf" ]; then
fi
# Get all files with .node extension from server folder
files=$(find $SEARCH_PATH -name "*.node" -not -path "*prebuilds*" -o -type f -executable -name "node")
files=$(find $SEARCH_PATH -name "*.node" -not -path "*prebuilds*" -not -path "*extensions/node_modules/@parcel/watcher*" -o -type f -executable -name "node")
echo "Verifying requirements for files: $files"