Engineering - Exploration to run tests in parallel (#149698)

This commit is contained in:
Ladislau Szomoru 2022-05-18 10:37:44 +02:00 committed by GitHub
parent 688c802459
commit d1f882f635
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 530 additions and 308 deletions

View file

@ -1,6 +1,15 @@
parameters:
- name: VSCODE_QUALITY
type: string
- name: VSCODE_RUN_UNIT_TESTS
type: boolean
default: true
- name: VSCODE_RUN_INTEGRATION_TESTS
type: boolean
default: true
- name: VSCODE_RUN_SMOKE_TESTS
type: boolean
default: true
steps:
- task: NodeTool@0
@ -165,91 +174,126 @@ steps:
VSCODE_ARCH=$(VSCODE_ARCH) DEBUG=electron-osx-sign* node build/darwin/sign.js
displayName: Set Hardened Entitlements
- script: |
set -e
./scripts/test.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
./scripts/test.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
- script: |
set -e
yarn test-node --build
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
yarn test-node --build
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
- script: |
set -e
DEBUG=*browser* yarn test-browser-no-install --sequential --build --browser chromium --browser webkit --tfs "Browser Unit Tests"
displayName: Run unit tests (Browser, Chromium & Webkit)
timeoutInMinutes: 30
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
DEBUG=*browser* yarn test-browser-no-install --sequential --build --browser chromium --browser webkit --tfs "Browser Unit Tests"
displayName: Run unit tests (Browser, Chromium & Webkit)
timeoutInMinutes: 30
- script: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
./scripts/test-integration.sh --build --tfs "Integration Tests"
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
yarn gulp \
compile-extension:css-language-features-server \
compile-extension:emmet \
compile-extension:git \
compile-extension:github-authentication \
compile-extension:html-language-features-server \
compile-extension:ipynb \
compile-extension:json-language-features-server \
compile-extension:markdown-language-features \
compile-extension-media \
compile-extension:microsoft-authentication \
compile-extension:typescript-language-features \
compile-extension:vscode-api-tests \
compile-extension:vscode-colorize-tests \
compile-extension:vscode-custom-editor-tests \
compile-extension:vscode-notebook-tests \
compile-extension:vscode-test-resolver
displayName: Build integration tests
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
./scripts/test-web-integration.sh --browser webkit
displayName: Run integration tests (Browser, Webkit)
timeoutInMinutes: 20
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
./scripts/test-integration.sh --build --tfs "Integration Tests"
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
./scripts/test-remote-integration.sh
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
./scripts/test-web-integration.sh --browser webkit
displayName: Run integration tests (Browser, Webkit)
timeoutInMinutes: 20
- script: |
set -e
ps -ef
displayName: Diagnostics before smoke test run
continueOnError: true
condition: succeededOrFailed()
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
./scripts/test-remote-integration.sh
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --web --tracing --headless
timeoutInMinutes: 20
displayName: Run smoke tests (Browser, Chromium)
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
ps -ef
displayName: Diagnostics before smoke test run
continueOnError: true
condition: succeededOrFailed()
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
yarn smoketest-no-compile --tracing --build "$APP_ROOT/$APP_NAME"
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --web --tracing --headless
timeoutInMinutes: 20
displayName: Run smoke tests (Browser, Chromium)
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
yarn smoketest-no-compile --tracing --build "$APP_ROOT/$APP_NAME"
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
- script: |
set -e
ps -ef
displayName: Diagnostics after smoke test run
continueOnError: true
condition: succeededOrFailed()
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
yarn gulp compile-extension:vscode-test-resolver
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
ps -ef
displayName: Diagnostics after smoke test run
continueOnError: true
condition: succeededOrFailed()
- task: PublishPipelineArtifact@0
inputs:

View file

@ -1,6 +1,15 @@
parameters:
- name: VSCODE_QUALITY
type: string
- name: VSCODE_RUN_UNIT_TESTS
type: boolean
default: true
- name: VSCODE_RUN_INTEGRATION_TESTS
type: boolean
default: true
- name: VSCODE_RUN_SMOKE_TESTS
type: boolean
default: true
steps:
- task: NodeTool@0
@ -221,104 +230,139 @@ steps:
stat $ELECTRON_ROOT/chrome-sandbox
displayName: Change setuid helper binary permission
- script: |
set -e
./scripts/test.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
./scripts/test.sh --build --tfs "Unit Tests"
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
yarn test-node --build
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
yarn test-node --build
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
DEBUG=*browser* yarn test-browser-no-install --build --browser chromium --tfs "Browser Unit Tests"
displayName: Run unit tests (Browser, Chromium)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- script: |
set -e
DEBUG=*browser* yarn test-browser-no-install --build --browser chromium --tfs "Browser Unit Tests"
displayName: Run unit tests (Browser, Chromium)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
INTEGRATION_TEST_APP_NAME="$APP_NAME" \
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
./scripts/test-integration.sh --build --tfs "Integration Tests"
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
yarn gulp \
compile-extension:css-language-features-server \
compile-extension:emmet \
compile-extension:git \
compile-extension:github-authentication \
compile-extension:html-language-features-server \
compile-extension:ipynb \
compile-extension:json-language-features-server \
compile-extension:markdown-language-features \
compile-extension-media \
compile-extension:microsoft-authentication \
compile-extension:typescript-language-features \
compile-extension:vscode-api-tests \
compile-extension:vscode-colorize-tests \
compile-extension:vscode-custom-editor-tests \
compile-extension:vscode-notebook-tests \
compile-extension:vscode-test-resolver
displayName: Build integration tests
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
./scripts/test-web-integration.sh --browser chromium
displayName: Run integration tests (Browser, Chromium)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
INTEGRATION_TEST_APP_NAME="$APP_NAME" \
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
./scripts/test-integration.sh --build --tfs "Integration Tests"
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
INTEGRATION_TEST_APP_NAME="$APP_NAME" \
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
./scripts/test-remote-integration.sh
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
./scripts/test-web-integration.sh --browser chromium
displayName: Run integration tests (Browser, Chromium)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
ps -ef
cat /proc/sys/fs/inotify/max_user_watches
lsof | wc -l
displayName: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles)
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- script: |
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
INTEGRATION_TEST_APP_NAME="$APP_NAME" \
INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
./scripts/test-remote-integration.sh
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
timeoutInMinutes: 20
displayName: Run smoke tests (Browser, Chromium)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
ps -ef
cat /proc/sys/fs/inotify/max_user_watches
lsof | wc -l
displayName: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles)
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
yarn smoketest-no-compile --tracing --build "$APP_PATH"
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
timeoutInMinutes: 20
displayName: Run smoke tests (Browser, Chromium)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
yarn smoketest-no-compile --tracing --build "$APP_PATH"
timeoutInMinutes: 20
displayName: Run smoke tests (Electron)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- script: |
set -e
ps -ef
cat /proc/sys/fs/inotify/max_user_watches
lsof | wc -l
displayName: Diagnostics after smoke test run (processes, max_user_watches, number of opened file handles)
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
yarn gulp compile-extension:vscode-test-resolver
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
timeoutInMinutes: 20
displayName: Run smoke tests (Remote)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- script: |
set -e
ps -ef
cat /proc/sys/fs/inotify/max_user_watches
lsof | wc -l
displayName: Diagnostics after smoke test run (processes, max_user_watches, number of opened file handles)
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- task: PublishPipelineArtifact@0
inputs:

View file

@ -60,7 +60,8 @@ stages:
- Compile
pool: vscode-1es-vscode-windows-2019
jobs:
- job: Windows
- job: WindowsUnitTests
displayName: Unit Tests
timeoutInMinutes: 120
variables:
VSCODE_ARCH: x64
@ -68,6 +69,33 @@ stages:
- template: win32/product-build-win32.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: true
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: false
- job: WindowsIntegrationTests
displayName: Integration Tests
timeoutInMinutes: 120
variables:
VSCODE_ARCH: x64
steps:
- template: win32/product-build-win32.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: true
VSCODE_RUN_SMOKE_TESTS: false
- job: WindowsSmokeTests
displayName: Smoke Tests
timeoutInMinutes: 120
variables:
VSCODE_ARCH: x64
steps:
- template: win32/product-build-win32.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: true
- stage: Linux
dependsOn:
@ -75,7 +103,8 @@ stages:
- LinuxServerDependencies
pool: vscode-1es-vscode-linux-18.04
jobs:
- job: Linuxx64
- job: Linuxx64UnitTest
displayName: Unit Tests
container: vscode-bionic-x64
variables:
VSCODE_ARCH: x64
@ -85,6 +114,37 @@ stages:
- template: linux/product-build-linux-client.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: true
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: false
- job: Linuxx64IntegrationTest
displayName: Integration Tests
container: vscode-bionic-x64
variables:
VSCODE_ARCH: x64
NPM_ARCH: x64
DISPLAY: ":10"
steps:
- template: linux/product-build-linux-client.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: true
VSCODE_RUN_SMOKE_TESTS: false
- job: Linuxx64SmokeTest
displayName: Smoke Tests
container: vscode-bionic-x64
variables:
VSCODE_ARCH: x64
NPM_ARCH: x64
DISPLAY: ":10"
steps:
- template: linux/product-build-linux-client.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: true
- stage: macOS
dependsOn:
@ -94,7 +154,8 @@ stages:
variables:
BUILDSECMON_OPT_IN: true
jobs:
- job: macOSTest
- job: macOSUnitTest
displayName: Unit Tests
timeoutInMinutes: 90
variables:
VSCODE_ARCH: x64
@ -102,3 +163,30 @@ stages:
- template: darwin/product-build-darwin-test.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: true
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: false
- job: macOSIntegrationTest
displayName: Integration Tests
timeoutInMinutes: 90
variables:
VSCODE_ARCH: x64
steps:
- template: darwin/product-build-darwin-test.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: true
VSCODE_RUN_SMOKE_TESTS: false
- job: macOSSmokeTest
displayName: Smoke Tests
timeoutInMinutes: 90
variables:
VSCODE_ARCH: x64
steps:
- template: darwin/product-build-darwin-test.yml
parameters:
VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
VSCODE_RUN_UNIT_TESTS: false
VSCODE_RUN_INTEGRATION_TESTS: false
VSCODE_RUN_SMOKE_TESTS: true

View file

@ -1,6 +1,15 @@
parameters:
- name: VSCODE_QUALITY
type: string
- name: VSCODE_RUN_UNIT_TESTS
type: boolean
default: true
- name: VSCODE_RUN_INTEGRATION_TESTS
type: boolean
default: true
- name: VSCODE_RUN_SMOKE_TESTS
type: boolean
default: true
steps:
- task: NodeTool@0
@ -184,105 +193,142 @@ steps:
displayName: Download Playwright
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn electron $(VSCODE_ARCH) }
exec { .\scripts\test.bat --build --tfs "Unit Tests" }
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn electron $(VSCODE_ARCH) }
exec { .\scripts\test.bat --build --tfs "Unit Tests" }
displayName: Run unit tests (Electron)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn test-node --build }
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn test-node --build }
displayName: Run unit tests (node.js)
timeoutInMinutes: 15
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn test-browser-no-install --sequential --build --browser chromium --browser firefox --tfs "Browser Unit Tests" }
displayName: Run unit tests (Browser, Chromium & Firefox)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn test-browser-no-install --sequential --build --browser chromium --browser firefox --tfs "Browser Unit Tests" }
displayName: Run unit tests (Browser, Chromium & Firefox)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
$AppNameShort = $AppProductJson.nameShort
exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-integration.bat --build --tfs "Integration Tests" }
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { yarn gulp `
compile-extension:css-language-features-server `
compile-extension:emmet `
compile-extension:git `
compile-extension:github-authentication `
compile-extension:html-language-features-server `
compile-extension:ipynb `
compile-extension:json-language-features-server `
compile-extension:markdown-language-features `
compile-extension-media `
compile-extension:microsoft-authentication `
compile-extension:typescript-language-features `
compile-extension:vscode-api-tests `
compile-extension:vscode-colorize-tests `
compile-extension:vscode-custom-editor-tests `
compile-extension:vscode-notebook-tests `
compile-extension:vscode-test-resolver `
}
displayName: Build integration tests
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"; .\scripts\test-web-integration.bat --browser firefox }
displayName: Run integration tests (Browser, Firefox)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- powershell: |
# Figure out the full absolute path of the product we just built
# including the remote server and configure the integration tests
# to run with these builds instead of running out of sources.
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
$AppNameShort = $AppProductJson.nameShort
exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-integration.bat --build --tfs "Integration Tests" }
displayName: Run integration tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
$AppNameShort = $AppProductJson.nameShort
exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-remote-integration.bat }
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
exec { $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"; .\scripts\test-web-integration.bat --browser firefox }
displayName: Run integration tests (Browser, Firefox)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
exec {.\build\azure-pipelines\win32\listprocesses.bat }
displayName: Diagnostics before smoke test run
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
$AppNameShort = $AppProductJson.nameShort
exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-remote-integration.bat }
displayName: Run integration tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --web --tracing --headless }
displayName: Run smoke tests (Browser, Chromium)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
exec {.\build\azure-pipelines\win32\listprocesses.bat }
displayName: Diagnostics before smoke test run
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
displayName: Run smoke tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --web --tracing --headless }
displayName: Run smoke tests (Browser, Chromium)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
displayName: Run smoke tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
displayName: Run smoke tests (Electron)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- powershell: |
. build/azure-pipelines/win32/exec.ps1
exec {.\build\azure-pipelines\win32\listprocesses.bat }
displayName: Diagnostics after smoke test run
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
exec { yarn gulp compile-extension:vscode-test-resolver }
exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
displayName: Run smoke tests (Remote)
timeoutInMinutes: 20
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
- ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
- powershell: |
. build/azure-pipelines/win32/exec.ps1
exec {.\build\azure-pipelines\win32\listprocesses.bat }
displayName: Diagnostics after smoke test run
continueOnError: true
condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
- task: PublishPipelineArtifact@0
inputs:

View file

@ -21,19 +21,19 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
:: Run from a built: need to compile all test extensions
:: because we run extension tests from their source folders
:: and the build bundles extensions into .build webpacked
call yarn gulp compile-extension:vscode-api-tests^
compile-extension:vscode-colorize-tests^
compile-extension:markdown-language-features^
compile-extension:typescript-language-features^
compile-extension:vscode-custom-editor-tests^
compile-extension:vscode-notebook-tests^
compile-extension:emmet^
compile-extension:css-language-features-server^
compile-extension:html-language-features-server^
compile-extension:json-language-features-server^
compile-extension:git^
compile-extension:ipynb^
compile-extension-media
:: call yarn gulp compile-extension:vscode-api-tests^
:: compile-extension:vscode-colorize-tests^
:: compile-extension:markdown-language-features^
:: compile-extension:typescript-language-features^
:: compile-extension:vscode-custom-editor-tests^
:: compile-extension:vscode-notebook-tests^
:: compile-extension:emmet^
:: compile-extension:css-language-features-server^
:: compile-extension:html-language-features-server^
:: compile-extension:json-language-features-server^
:: compile-extension:git^
:: compile-extension:ipynb^
:: compile-extension-media
:: Configuration for more verbose output
set VSCODE_CLI=1

View file

@ -30,19 +30,19 @@ else
# Run from a built: need to compile all test extensions
# because we run extension tests from their source folders
# and the build bundles extensions into .build webpacked
yarn gulp compile-extension:vscode-api-tests \
compile-extension:vscode-colorize-tests \
compile-extension:vscode-custom-editor-tests \
compile-extension:vscode-notebook-tests \
compile-extension:markdown-language-features \
compile-extension:typescript-language-features \
compile-extension:emmet \
compile-extension:css-language-features-server \
compile-extension:html-language-features-server \
compile-extension:json-language-features-server \
compile-extension:git \
compile-extension:ipynb \
compile-extension-media
# yarn gulp compile-extension:vscode-api-tests \
# compile-extension:vscode-colorize-tests \
# compile-extension:vscode-custom-editor-tests \
# compile-extension:vscode-notebook-tests \
# compile-extension:markdown-language-features \
# compile-extension:typescript-language-features \
# compile-extension:emmet \
# compile-extension:css-language-features-server \
# compile-extension:html-language-features-server \
# compile-extension:json-language-features-server \
# compile-extension:git \
# compile-extension:ipynb \
# compile-extension-media
# Configuration for more verbose output
export VSCODE_CLI=1

View file

@ -53,10 +53,10 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
:: Run from a built: need to compile all test extensions
:: because we run extension tests from their source folders
:: and the build bundles extensions into .build webpacked
call yarn gulp compile-extension:vscode-api-tests^
compile-extension:microsoft-authentication^
compile-extension:github-authentication^
compile-extension:vscode-test-resolver
:: call yarn gulp compile-extension:vscode-api-tests^
:: compile-extension:microsoft-authentication^
:: compile-extension:github-authentication^
:: compile-extension:vscode-test-resolver
:: Configuration for more verbose output
set VSCODE_CLI=1

View file

@ -47,16 +47,16 @@ else
# Run from a built: need to compile all test extensions
# because we run extension tests from their source folders
# and the build bundles extensions into .build webpacked
yarn gulp compile-extension:vscode-api-tests \
compile-extension:vscode-test-resolver \
compile-extension:markdown-language-features \
compile-extension:typescript-language-features \
compile-extension:emmet \
compile-extension:git \
compile-extension:ipynb \
compile-extension:microsoft-authentication \
compile-extension:github-authentication \
compile-extension-media
# yarn gulp compile-extension:vscode-api-tests \
# compile-extension:vscode-test-resolver \
# compile-extension:markdown-language-features \
# compile-extension:typescript-language-features \
# compile-extension:emmet \
# compile-extension:git \
# compile-extension:ipynb \
# compile-extension:microsoft-authentication \
# compile-extension:github-authentication \
# compile-extension-media
# Configuration for more verbose output
export VSCODE_CLI=1

View file

@ -25,12 +25,12 @@ if "%VSCODE_REMOTE_SERVER_PATH%"=="" (
:: Run from a built: need to compile all test extensions
:: because we run extension tests from their source folders
:: and the build bundles extensions into .build webpacked
call yarn gulp compile-extension:vscode-api-tests^
compile-extension:markdown-language-features^
compile-extension:typescript-language-features^
compile-extension:emmet^
compile-extension:git^
compile-extension-media
:: call yarn gulp compile-extension:vscode-api-tests^
:: compile-extension:markdown-language-features^
:: compile-extension:typescript-language-features^
:: compile-extension:emmet^
:: compile-extension:git^
:: compile-extension-media
)
if not exist ".\test\integration\browser\out\index.js" (

View file

@ -19,13 +19,13 @@ else
# Run from a built: need to compile all test extensions
# because we run extension tests from their source folders
# and the build bundles extensions into .build webpacked
yarn gulp compile-extension:vscode-api-tests \
compile-extension:markdown-language-features \
compile-extension:typescript-language-features \
compile-extension:emmet \
compile-extension:git \
compile-extension:ipynb \
compile-extension-media
# yarn gulp compile-extension:vscode-api-tests \
# compile-extension:markdown-language-features \
# compile-extension:typescript-language-features \
# compile-extension:emmet \
# compile-extension:git \
# compile-extension:ipynb \
# compile-extension-media
fi
if [ ! -e 'test/integration/browser/out/index.js' ];then