diff --git a/.github/skills/integration-tests/SKILL.md b/.github/skills/integration-tests/SKILL.md new file mode 100644 index 00000000000..e8428fac898 --- /dev/null +++ b/.github/skills/integration-tests/SKILL.md @@ -0,0 +1,122 @@ +--- +name: integration-tests +description: Use when running integration tests in the VS Code repo. Covers scripts/test-integration.sh (macOS/Linux) and scripts/test-integration.bat (Windows), their supported arguments for filtering, and the difference between node.js integration tests and extension host tests. +--- + +# Running Integration Tests + +Integration tests in VS Code are split into two categories: + +1. **Node.js integration tests** - files ending in `.integrationTest.ts` under `src/`. These run in Electron via the same Mocha runner as unit tests. +2. **Extension host tests** - tests embedded in built-in extensions under `extensions/` (API tests, Git tests, TypeScript tests, etc.). These launch a full VS Code instance with `--extensionDevelopmentPath`. + +## Scripts + +- **macOS / Linux:** `./scripts/test-integration.sh [options]` +- **Windows:** `.\scripts\test-integration.bat [options]` + +When run **without filters**, both scripts execute all node.js integration tests followed by all extension host tests. + +When run **with `--run` or `--runGlob`** (without `--suite`), only the node.js integration tests are run and the filter is applied. Extension host tests are skipped since these filters are node.js-specific. + +When run **with `--grep` alone** (no `--run`, `--runGlob`, or `--suite`), all tests are run -- both node.js integration tests and all extension host suites -- with the grep pattern forwarded to every test runner. + +When run **with `--suite`**, only the matching extension host test suites are run. Node.js integration tests are skipped. Combine `--suite` with `--grep` to filter individual tests within the selected suites. + +## Options + +### `--run ` - Run tests from a specific file + +Accepts a **source file path** (starting with `src/`). Works identically to `scripts/test.sh --run`. + +```bash +./scripts/test-integration.sh --run src/vs/workbench/services/search/test/browser/search.integrationTest.ts +``` + +### `--runGlob ` (aliases: `--glob`, `--runGrep`) - Select test files by path + +Selects which test **files** to load by matching compiled `.js` file paths against a glob pattern. Overrides the default `**/*.integrationTest.js` glob. Only applies to node.js integration tests (extension host tests are skipped). + +```bash +./scripts/test-integration.sh --runGlob "**/search/**/*.integrationTest.js" +``` + +### `--grep ` (aliases: `-g`, `-f`) - Filter test cases by name + +Filters which **test cases** run by matching against their test titles (e.g. `describe`/`test` names). When used alone, the grep is applied to both node.js integration tests and all extension host suites. When combined with `--suite`, only the matched suites run with the grep. + +```bash +./scripts/test-integration.sh --grep "TextSearchProvider" +``` + +### `--suite ` - Run specific extension host test suites + +Runs only the extension host test suites whose name matches the pattern. Supports comma-separated values and shell glob patterns (on macOS/Linux). Node.js integration tests are skipped. + +Available suite names: `api-folder`, `api-workspace`, `colorize`, `terminal-suggest`, `typescript`, `markdown`, `emmet`, `git`, `git-base`, `ipynb`, `notebook-renderers`, `configuration-editing`, `github-authentication`, `css`, `html`. + +```bash +# Run only Git extension tests +./scripts/test-integration.sh --suite git + +# Run API folder and workspace tests (glob, macOS/Linux only) +./scripts/test-integration.sh --suite 'api*' + +# Run multiple specific suites +./scripts/test-integration.sh --suite 'git,emmet,typescript' + +# Filter tests within a suite by name +./scripts/test-integration.sh --suite api-folder --grep 'should open' +``` + +### `--help`, `-h` - Show help + +```bash +./scripts/test-integration.sh --help +``` + +### Other options + +All other options (e.g. `--timeout`, `--coverage`, `--reporter`) are forwarded to the underlying `scripts/test.sh` runner for node.js integration tests. These extra options are **not** forwarded to extension host suites when using `--suite`. + +## Examples + +```bash +# Run all integration tests (node.js + extension host) +./scripts/test-integration.sh + +# Run a single integration test file +./scripts/test-integration.sh --run src/vs/workbench/services/search/test/browser/search.integrationTest.ts + +# Run integration tests matching a grep pattern +./scripts/test-integration.sh --grep "TextSearchProvider" + +# Run integration tests under a specific area +./scripts/test-integration.sh --runGlob "**/workbench/**/*.integrationTest.js" + +# Run only Git extension host tests +./scripts/test-integration.sh --suite git + +# Run API folder + workspace extension tests (glob) +./scripts/test-integration.sh --suite 'api*' + +# Run multiple extension test suites +./scripts/test-integration.sh --suite 'git,typescript,emmet' + +# Grep for specific tests in the API folder suite +./scripts/test-integration.sh --suite api-folder --grep 'should open' + +# Combine file and grep +./scripts/test-integration.sh --run src/vs/workbench/services/search/test/browser/search.integrationTest.ts --grep "should search" +``` + +## Compilation requirement + +Tests run against compiled JavaScript output. Ensure the `VS Code - Build` watch task is running or that compilation has completed before running tests. + +## Distinction from unit tests + +- **Unit tests** (`.test.ts`) → use `scripts/test.sh` or the `runTests` tool +- **Integration tests** (`.integrationTest.ts` and extension tests) → use `scripts/test-integration.sh` + +Do **not** mix these up: `scripts/test.sh` will not find integration test files unless you explicitly pass `--runGlob **/*.integrationTest.js`, and `scripts/test-integration.sh` is not intended for `.test.ts` files. diff --git a/.github/skills/unit-tests/SKILL.md b/.github/skills/unit-tests/SKILL.md index f2a8b66c5a3..18f0290877c 100644 --- a/.github/skills/unit-tests/SKILL.md +++ b/.github/skills/unit-tests/SKILL.md @@ -80,7 +80,7 @@ Override the default Mocha timeout for long-running tests. ### Integration tests -Integration tests (files ending in `.integrationTest.ts` or located in `extensions/`) are **not run** by `scripts/test.sh`. Use `scripts/test-integration.sh` (or `scripts/test-integration.bat`) instead. +Integration tests (files ending in `.integrationTest.ts` or located in `extensions/`) are **not run** by `scripts/test.sh`. Use `scripts/test-integration.sh` (or `scripts/test-integration.bat`) instead. See the `integration-tests` skill for details on filtering and running specific integration test files. ### Compilation requirement diff --git a/extensions/css-language-features/server/test/index.js b/extensions/css-language-features/server/test/index.js index 0c9be2d9710..f6d750ea756 100644 --- a/extensions/css-language-features/server/test/index.js +++ b/extensions/css-language-features/server/test/index.js @@ -15,6 +15,10 @@ const options = { timeout: 60000 }; +if (process.env.MOCHA_GREP) { + options.grep = process.env.MOCHA_GREP; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY || process.env.GITHUB_WORKSPACE) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/html-language-features/server/test/index.js b/extensions/html-language-features/server/test/index.js index 93ffe7d6c09..aae624e8ac4 100644 --- a/extensions/html-language-features/server/test/index.js +++ b/extensions/html-language-features/server/test/index.js @@ -15,6 +15,10 @@ const options = { timeout: 60000 }; +if (process.env.MOCHA_GREP) { + options.grep = process.env.MOCHA_GREP; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY || process.env.GITHUB_WORKSPACE) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/scripts/test-integration.bat b/scripts/test-integration.bat index 2be5bfef0a0..5c68dbec12e 100644 --- a/scripts/test-integration.bat +++ b/scripts/test-integration.bat @@ -1,11 +1,90 @@ @echo off setlocal -pushd %~dp0\.. +:: Capture script directory before any call :label changes %~dp0 +set "SCRIPT_DIR=%~dp0" + +pushd %SCRIPT_DIR%\.. + +:: Parse arguments for help and filters +set "HAS_FILTER=" +set "RUN_FILE=" +set "RUN_GLOB=" +set "GREP_PATTERN=" +set "SUITE_FILTER=" +set "SHOW_HELP=" + +:parse_args +if "%~1"=="" goto done_parsing +if /i "%~1"=="--help" (set SHOW_HELP=1& shift & goto parse_args) +if /i "%~1"=="-h" (set SHOW_HELP=1& shift & goto parse_args) +if /i "%~1"=="--run" (set "RUN_FILE=%~2"& set HAS_FILTER=1& shift & shift & goto parse_args) +if /i "%~1"=="--grep" (set "GREP_PATTERN=%~2"& shift & shift & goto parse_args) +if /i "%~1"=="-g" (set "GREP_PATTERN=%~2"& shift & shift & goto parse_args) +if /i "%~1"=="-f" (set "GREP_PATTERN=%~2"& shift & shift & goto parse_args) +if /i "%~1"=="--runGlob" (set "RUN_GLOB=%~2"& set HAS_FILTER=1& shift & shift & goto parse_args) +if /i "%~1"=="--glob" (set "RUN_GLOB=%~2"& set HAS_FILTER=1& shift & shift & goto parse_args) +if /i "%~1"=="--runGrep" (set "RUN_GLOB=%~2"& set HAS_FILTER=1& shift & shift & goto parse_args) +if /i "%~1"=="--suite" (set "SUITE_FILTER=%~2"& shift & shift & goto parse_args) +shift +goto parse_args +:done_parsing + +if defined SHOW_HELP ( + echo Usage: %~nx0 [options] + echo. + echo Runs integration tests. When no filters are given, all integration tests + echo ^(node.js integration tests + extension host tests^) are run. + echo. + echo --run and --runGlob select which node.js integration test files to load. + echo Extension host tests are skipped when these options are used. + echo. + echo --grep filters test cases by name across all test runners. When used alone, + echo the pattern is applied to both node.js integration tests and all extension + echo host suites. When combined with --suite, only the selected suites are run. + echo. + echo --suite selects which extension host test suites to run. + echo Node.js integration tests are skipped when this option is used. + echo. + echo Options: + echo --run ^ run tests from a specific file ^(src/ path^) + echo --runGlob, --glob ^ select test files by path glob ^(e.g. '**\*.integrationTest.js'^) + echo --grep, -g, -f ^ filter test cases by name ^(matched against test titles^) + echo --suite ^ run only matching extension host test suites + echo supports comma-separated list + echo --help, -h show this help + echo. + echo Available suites: + echo api-folder, api-workspace, colorize, terminal-suggest, typescript, + echo markdown, emmet, git, git-base, ipynb, notebook-renderers, + echo configuration-editing, github-authentication, css, html + echo. + echo All other options are forwarded to the node.js test runner ^(see scripts\test.bat --help^). + echo Note: extra options are not forwarded to extension host suites ^(--suite mode^). + echo. + echo Examples: + echo %~nx0 + echo %~nx0 --run src\vs\editor\test\browser\controller.integrationTest.ts + echo %~nx0 --grep "some test name" + echo %~nx0 --runGlob "**\*.integrationTest.js" + echo %~nx0 --suite git # run only Git tests + echo %~nx0 --suite "api-folder,api-workspace" # run multiple suites + echo %~nx0 --suite api-folder --grep "some test" # grep within a suite + exit /b 0 +) set VSCODEUSERDATADIR=%TEMP%\vscodeuserfolder-%RANDOM%-%TIME:~6,2% -set VSCODECRASHDIR=%~dp0\..\.build\crashes -set VSCODELOGSDIR=%~dp0\..\.build\logs\integration-tests +set VSCODECRASHDIR=%SCRIPT_DIR%\..\.build\crashes +set VSCODELOGSDIR=%SCRIPT_DIR%\..\.build\logs\integration-tests + +:: Seed user settings to disable OS notifications (dock bounce, toast, etc.) +if not exist "%VSCODEUSERDATADIR%\User" mkdir "%VSCODEUSERDATADIR%\User" +( +echo { +echo "chat.notifyWindowOnConfirmation": "off", +echo "chat.notifyWindowOnResponseReceived": "off" +echo } +) > "%VSCODEUSERDATADIR%\User\settings.json" :: Figure out which Electron to use for running tests if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" ( @@ -25,105 +104,213 @@ echo Storing crash reports into '%VSCODECRASHDIR%'. echo Storing log files into '%VSCODELOGSDIR%'. -:: Unit tests +:: Validate --suite filter matches at least one known suite +if defined SUITE_FILTER ( + set "_any_match=" + for %%s in (api-folder api-workspace colorize terminal-suggest typescript markdown emmet git git-base ipynb notebook-renderers configuration-editing github-authentication css html) do ( + call :should_run_suite %%s && set "_any_match=1" + ) + if not defined _any_match ( + echo Error: no suites match filter '%SUITE_FILTER%' + echo Available suites: api-folder api-workspace colorize terminal-suggest typescript markdown emmet git git-base ipynb notebook-renderers configuration-editing github-authentication css html + exit /b 1 + ) +) + +:: Node.js integration tests + +if defined SUITE_FILTER goto skip_nodejs_tests echo. echo ### node.js integration tests -call .\scripts\test.bat --runGlob **\*.integrationTest.js %* +if defined RUN_GLOB ( + call .\scripts\test.bat %* +) else if defined RUN_FILE ( + call .\scripts\test.bat %* +) else ( + call .\scripts\test.bat --runGlob **\*.integrationTest.js %* +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_nodejs_tests + +:: Skip extension host tests when a non-suite filter is active +if not defined SUITE_FILTER if defined HAS_FILTER ( + echo. + echo Filter active, skipping extension host tests. + rmdir /s /q "%VSCODEUSERDATADIR%" 2>nul + exit /b 0 +) :: Tests in the extension host +:: Forward grep pattern to extension test runners +if defined GREP_PATTERN set "MOCHA_GREP=%GREP_PATTERN%" + set API_TESTS_EXTRA_ARGS=--disable-telemetry --disable-experiments --skip-welcome --skip-release-notes --crash-reporter-directory=%VSCODECRASHDIR% --logsPath=%VSCODELOGSDIR% --no-cached-data --disable-updates --use-inmemory-secretstorage --disable-extensions --disable-workspace-trust --user-data-dir=%VSCODEUSERDATADIR% +call :should_run_suite api-folder || goto skip_api_folder echo. echo ### API tests (folder) -call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\vscode-api-tests\testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%~dp0\..\extensions\vscode-api-tests --extensionTestsPath=%~dp0\..\extensions\vscode-api-tests\out\singlefolder-tests %API_TESTS_EXTRA_ARGS% +call "%INTEGRATION_TEST_ELECTRON_PATH%" %SCRIPT_DIR%\..\extensions\vscode-api-tests\testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%SCRIPT_DIR%\..\extensions\vscode-api-tests --extensionTestsPath=%SCRIPT_DIR%\..\extensions\vscode-api-tests\out\singlefolder-tests %API_TESTS_EXTRA_ARGS% if %errorlevel% neq 0 exit /b %errorlevel% +:skip_api_folder +call :should_run_suite api-workspace || goto skip_api_workspace echo. echo ### API tests (workspace) -call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\vscode-api-tests\testworkspace.code-workspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%~dp0\..\extensions\vscode-api-tests --extensionTestsPath=%~dp0\..\extensions\vscode-api-tests\out\workspace-tests %API_TESTS_EXTRA_ARGS% +call "%INTEGRATION_TEST_ELECTRON_PATH%" %SCRIPT_DIR%\..\extensions\vscode-api-tests\testworkspace.code-workspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%SCRIPT_DIR%\..\extensions\vscode-api-tests --extensionTestsPath=%SCRIPT_DIR%\..\extensions\vscode-api-tests\out\workspace-tests %API_TESTS_EXTRA_ARGS% if %errorlevel% neq 0 exit /b %errorlevel% +:skip_api_workspace +call :should_run_suite colorize || goto skip_colorize echo. echo ### Colorize tests -call npm run test-extension -- -l vscode-colorize-tests +if defined GREP_PATTERN ( + call npm run test-extension -- -l vscode-colorize-tests --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l vscode-colorize-tests +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_colorize +call :should_run_suite terminal-suggest || goto skip_terminal_suggest echo. echo ### Terminal Suggest tests -call npm run test-extension -- -l terminal-suggest --enable-proposed-api=vscode.vscode-api-tests +if defined GREP_PATTERN ( + call npm run test-extension -- -l terminal-suggest --enable-proposed-api=vscode.vscode-api-tests --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l terminal-suggest --enable-proposed-api=vscode.vscode-api-tests +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_terminal_suggest +call :should_run_suite typescript || goto skip_typescript echo. echo ### TypeScript tests -call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\typescript-language-features\test-workspace --extensionDevelopmentPath=%~dp0\..\extensions\typescript-language-features --extensionTestsPath=%~dp0\..\extensions\typescript-language-features\out\test\unit %API_TESTS_EXTRA_ARGS% +call "%INTEGRATION_TEST_ELECTRON_PATH%" %SCRIPT_DIR%\..\extensions\typescript-language-features\test-workspace --extensionDevelopmentPath=%SCRIPT_DIR%\..\extensions\typescript-language-features --extensionTestsPath=%SCRIPT_DIR%\..\extensions\typescript-language-features\out\test\unit %API_TESTS_EXTRA_ARGS% if %errorlevel% neq 0 exit /b %errorlevel% +:skip_typescript +call :should_run_suite markdown || goto skip_markdown echo. echo ### Markdown tests -call npm run test-extension -- -l markdown-language-features +if defined GREP_PATTERN ( + call npm run test-extension -- -l markdown-language-features --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l markdown-language-features +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_markdown +call :should_run_suite emmet || goto skip_emmet echo. echo ### Emmet tests -call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\emmet\test-workspace --extensionDevelopmentPath=%~dp0\..\extensions\emmet --extensionTestsPath=%~dp0\..\extensions\emmet\out\test %API_TESTS_EXTRA_ARGS% +call "%INTEGRATION_TEST_ELECTRON_PATH%" %SCRIPT_DIR%\..\extensions\emmet\test-workspace --extensionDevelopmentPath=%SCRIPT_DIR%\..\extensions\emmet --extensionTestsPath=%SCRIPT_DIR%\..\extensions\emmet\out\test %API_TESTS_EXTRA_ARGS% if %errorlevel% neq 0 exit /b %errorlevel% +:skip_emmet +call :should_run_suite git || goto skip_git echo. echo ### Git tests for /f "delims=" %%i in ('node -p "require('fs').realpathSync.native(require('os').tmpdir())"') do set TEMPDIR=%%i set GITWORKSPACE=%TEMPDIR%\git-%RANDOM% mkdir %GITWORKSPACE% -call "%INTEGRATION_TEST_ELECTRON_PATH%" %GITWORKSPACE% --extensionDevelopmentPath=%~dp0\..\extensions\git --extensionTestsPath=%~dp0\..\extensions\git\out\test %API_TESTS_EXTRA_ARGS% +call "%INTEGRATION_TEST_ELECTRON_PATH%" %GITWORKSPACE% --extensionDevelopmentPath=%SCRIPT_DIR%\..\extensions\git --extensionTestsPath=%SCRIPT_DIR%\..\extensions\git\out\test %API_TESTS_EXTRA_ARGS% if %errorlevel% neq 0 exit /b %errorlevel% +:skip_git +call :should_run_suite git-base || goto skip_git_base echo. echo ### Git Base tests -call npm run test-extension -- -l git-base +if defined GREP_PATTERN ( + call npm run test-extension -- -l git-base --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l git-base +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_git_base +call :should_run_suite ipynb || goto skip_ipynb echo. echo ### Ipynb tests -call npm run test-extension -- -l ipynb +if defined GREP_PATTERN ( + call npm run test-extension -- -l ipynb --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l ipynb +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_ipynb +call :should_run_suite notebook-renderers || goto skip_notebook_renderers echo. echo ### Notebook Output tests -call npm run test-extension -- -l notebook-renderers +if defined GREP_PATTERN ( + call npm run test-extension -- -l notebook-renderers --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l notebook-renderers +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_notebook_renderers +call :should_run_suite configuration-editing || goto skip_configuration_editing echo. echo ### Configuration editing tests -set CFWORKSPACE=%TEMPDIR%\cf-%RANDOM% -mkdir %CFWORKSPACE% -call npm run test-extension -- -l configuration-editing +if defined GREP_PATTERN ( + call npm run test-extension -- -l configuration-editing --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l configuration-editing +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_configuration_editing +call :should_run_suite github-authentication || goto skip_github_authentication echo. echo ### GitHub Authentication tests -call npm run test-extension -- -l github-authentication +if defined GREP_PATTERN ( + call npm run test-extension -- -l github-authentication --grep "%GREP_PATTERN%" +) else ( + call npm run test-extension -- -l github-authentication +) if %errorlevel% neq 0 exit /b %errorlevel% +:skip_github_authentication :: Tests standalone (CommonJS) +call :should_run_suite css || goto skip_css echo. echo ### CSS tests -call %~dp0\node-electron.bat %~dp0\..\extensions\css-language-features/server/test/index.js +call %SCRIPT_DIR%\node-electron.bat %SCRIPT_DIR%\..\extensions\css-language-features/server/test/index.js if %errorlevel% neq 0 exit /b %errorlevel% +:skip_css +call :should_run_suite html || goto skip_html echo. echo ### HTML tests -call %~dp0\node-electron.bat %~dp0\..\extensions\html-language-features/server/test/index.js +call %SCRIPT_DIR%\node-electron.bat %SCRIPT_DIR%\..\extensions\html-language-features/server/test/index.js if %errorlevel% neq 0 exit /b %errorlevel% +:skip_html :: Cleanup -rmdir /s /q %VSCODEUSERDATADIR% +rmdir /s /q "%VSCODEUSERDATADIR%" popd +goto :end + +:: Subroutine: check whether a suite should run based on SUITE_FILTER. +:: Returns errorlevel 0 if the suite should run, 1 if it should be skipped. +:should_run_suite +if not defined SUITE_FILTER exit /b 0 +set "_suite_name=%~1" +:: Replace commas with spaces so for-loop tokenizes correctly +set "_filter=%SUITE_FILTER:,= %" +for %%p in (%_filter%) do ( + if /i "%%p"=="%_suite_name%" exit /b 0 +) +exit /b 1 + +:end endlocal diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index e3c391004f8..9c542509142 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -8,11 +8,126 @@ else ROOT=$(dirname $(dirname $(readlink -f $0))) fi +cd "$ROOT" + +# Parse arguments +EXTRA_ARGS=() +RUN_FILE="" +RUN_GLOB="" +GREP_PATTERN="" +SUITE_FILTER="" +HELP=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --help|-h) + HELP=true + shift + ;; + --run) + RUN_FILE="$2" + EXTRA_ARGS+=("$1" "$2") + shift 2 + ;; + --grep|-g|-f) + GREP_PATTERN="$2" + EXTRA_ARGS+=("$1" "$2") + shift 2 + ;; + --runGlob|--glob|--runGrep) + RUN_GLOB="$2" + EXTRA_ARGS+=("$1" "$2") + shift 2 + ;; + --suite) + SUITE_FILTER="$2" + shift 2 + ;; + *) + EXTRA_ARGS+=("$1") + shift + ;; + esac +done + +# Known suite names (used for help text and validation) +KNOWN_SUITES="api-folder api-workspace colorize terminal-suggest typescript markdown emmet git git-base ipynb notebook-renderers configuration-editing github-authentication css html" + +if $HELP; then + echo "Usage: $0 [options]" + echo "" + echo "Runs integration tests. When no filters are given, all integration tests" + echo "(node.js integration tests + extension host tests) are run." + echo "" + echo "--run and --runGlob select which node.js integration test files to load." + echo "Extension host tests are skipped when these options are used." + echo "" + echo "--grep filters test cases by name across all test runners. When used alone," + echo "the pattern is applied to both node.js integration tests and all extension" + echo "host suites. When combined with --suite, only the selected suites are run." + echo "" + echo "--suite selects which extension host test suites to run." + echo "Node.js integration tests are skipped when this option is used." + echo "" + echo "Options:" + echo " --run run tests from a specific file (src/ path)" + echo " --runGlob, --glob select test files by path glob (e.g. '**/editor/**/*.integrationTest.js')" + echo " --grep, -g, -f filter test cases by name (matched against test titles)" + echo " --suite run only matching extension host test suites" + echo " supports comma-separated list and glob patterns" + echo " --help, -h show this help" + echo "" + echo "Available suites: $KNOWN_SUITES" + echo "" + echo "All other options are forwarded to the node.js test runner (see scripts/test.sh --help)." + echo "Note: extra options are not forwarded to extension host suites (--suite mode)." + echo "" + echo "Examples:" + echo " $0 # run all integration tests" + echo " $0 --run src/vs/editor/test/browser/controller.integrationTest.ts" + echo " $0 --grep 'some test name'" + echo " $0 --runGlob '**/editor/**/*.integrationTest.js'" + echo " $0 --suite git # run only Git tests" + echo " $0 --suite 'api*' # run API folder + workspace tests" + echo " $0 --suite 'git,emmet,typescript' # run multiple suites" + echo " $0 --suite api-folder --grep 'some test' # grep within a suite" + exit 0 +fi + +HAS_FILTER=false +if [[ -n "$RUN_FILE" || -n "$RUN_GLOB" ]]; then + HAS_FILTER=true +fi + +# Check whether a given suite name matches the --suite filter. +# Supports comma-separated patterns with shell globbing (e.g. "git*,api*"). +should_run_suite() { + if [[ -z "$SUITE_FILTER" ]]; then + return 0 + fi + IFS=',' read -ra PATTERNS <<< "$SUITE_FILTER" + for pattern in "${PATTERNS[@]}"; do + pattern="${pattern## }" # trim leading spaces + pattern="${pattern%% }" # trim trailing spaces + if [[ "$1" == $pattern ]]; then + return 0 + fi + done + return 1 +} + VSCODEUSERDATADIR=`mktemp -d 2>/dev/null` VSCODECRASHDIR=$ROOT/.build/crashes VSCODELOGSDIR=$ROOT/.build/logs/integration-tests -cd $ROOT +# Seed user settings to disable OS notifications (dock bounce, toast, etc.) +mkdir -p "$VSCODEUSERDATADIR/User" +cat > "$VSCODEUSERDATADIR/User/settings.json" </dev/null) --extensionDevelopmentPath=$ROOT/extensions/git --extensionTestsPath=$ROOT/extensions/git/out/test $API_TESTS_EXTRA_ARGS kill_app +fi +if should_run_suite git-base; then echo echo "### Git Base tests" echo -npm run test-extension -- -l git-base +npm run test-extension -- -l git-base "${GREP_ARGS[@]}" kill_app +fi +if should_run_suite ipynb; then echo echo "### Ipynb tests" echo -npm run test-extension -- -l ipynb +npm run test-extension -- -l ipynb "${GREP_ARGS[@]}" kill_app +fi +if should_run_suite notebook-renderers; then echo echo "### Notebook Output tests" echo -npm run test-extension -- -l notebook-renderers +npm run test-extension -- -l notebook-renderers "${GREP_ARGS[@]}" kill_app +fi +if should_run_suite configuration-editing; then echo echo "### Configuration editing tests" echo -npm run test-extension -- -l configuration-editing +npm run test-extension -- -l configuration-editing "${GREP_ARGS[@]}" kill_app +fi +if should_run_suite github-authentication; then echo echo "### GitHub Authentication tests" echo -npm run test-extension -- -l github-authentication +npm run test-extension -- -l github-authentication "${GREP_ARGS[@]}" kill_app +fi # Tests standalone (CommonJS) +if should_run_suite css; then echo echo "### CSS tests" echo -cd $ROOT/extensions/css-language-features/server && $ROOT/scripts/node-electron.sh test/index.js +cd "$ROOT/extensions/css-language-features/server" && "$ROOT/scripts/node-electron.sh" test/index.js +fi +if should_run_suite html; then echo echo "### HTML tests" echo -cd $ROOT/extensions/html-language-features/server && $ROOT/scripts/node-electron.sh test/index.js +cd "$ROOT/extensions/html-language-features/server" && "$ROOT/scripts/node-electron.sh" test/index.js +fi # Cleanup -rm -rf $VSCODEUSERDATADIR +rm -rf -- "$VSCODEUSERDATADIR" diff --git a/test/integration/electron/testrunner.js b/test/integration/electron/testrunner.js index 8c55a40e4df..93f518da3b7 100644 --- a/test/integration/electron/testrunner.js +++ b/test/integration/electron/testrunner.js @@ -24,6 +24,9 @@ let mocha = new Mocha({ }); exports.configure = function configure(opts) { + if (process.env.MOCHA_GREP) { + opts.grep = process.env.MOCHA_GREP; + } mocha = new Mocha(opts); };