From 06730e3e29c3d8a798a4402f116cbff47a1a7bc1 Mon Sep 17 00:00:00 2001 From: Ladislau Szomoru <3372902+lszomoru@users.noreply.github.com> Date: Fri, 21 Aug 2020 15:05:15 +0200 Subject: [PATCH] Azure DevOps pipeline - generate unique test result names (#105148) * Set suite name based on environment variable --- extensions/emmet/src/test/index.ts | 14 ++++++++++++-- extensions/git/src/test/index.ts | 14 ++++++++++++-- .../markdown-language-features/src/test/index.ts | 14 ++++++++++++-- .../src/singlefolder-tests/index.ts | 14 ++++++++++++-- .../vscode-api-tests/src/workspace-tests/index.ts | 14 ++++++++++++-- extensions/vscode-notebook-tests/src/index.ts | 14 ++++++++++++-- test/integration/browser/src/index.ts | 9 ++++++--- 7 files changed, 78 insertions(+), 15 deletions(-) diff --git a/extensions/emmet/src/test/index.ts b/extensions/emmet/src/test/index.ts index dc77f1beb82..3aeda9dfa42 100644 --- a/extensions/emmet/src/test/index.ts +++ b/extensions/emmet/src/test/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Emmet Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Emmet Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Emmet Tests'; +} else { + suite = 'Integration Emmet Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/git/src/test/index.ts b/extensions/git/src/test/index.ts index a3588b2238c..8773f772e62 100644 --- a/extensions/git/src/test/index.ts +++ b/extensions/git/src/test/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Git Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Git Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Git Tests'; +} else { + suite = 'Integration Git Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/markdown-language-features/src/test/index.ts b/extensions/markdown-language-features/src/test/index.ts index a6d471ee2c1..0eb9bc92487 100644 --- a/extensions/markdown-language-features/src/test/index.ts +++ b/extensions/markdown-language-features/src/test/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Markdown Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Markdown Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Markdown Tests'; +} else { + suite = 'Integration Markdown Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/vscode-api-tests/src/singlefolder-tests/index.ts b/extensions/vscode-api-tests/src/singlefolder-tests/index.ts index 3bd54c52ef6..ea50f38b1b9 100644 --- a/extensions/vscode-api-tests/src/singlefolder-tests/index.ts +++ b/extensions/vscode-api-tests/src/singlefolder-tests/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Single Folder Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Single Folder Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Single Folder Tests'; +} else { + suite = 'Integration Single Folder Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/vscode-api-tests/src/workspace-tests/index.ts b/extensions/vscode-api-tests/src/workspace-tests/index.ts index 7f032ab0b5e..9486d8ed3e5 100644 --- a/extensions/vscode-api-tests/src/workspace-tests/index.ts +++ b/extensions/vscode-api-tests/src/workspace-tests/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Workspace Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Workspace Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Workspace Tests'; +} else { + suite = 'Integration Workspace Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/vscode-notebook-tests/src/index.ts b/extensions/vscode-notebook-tests/src/index.ts index 2da40bed655..293c02db743 100644 --- a/extensions/vscode-notebook-tests/src/index.ts +++ b/extensions/vscode-notebook-tests/src/index.ts @@ -6,14 +6,24 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Notebook Tests'; - const options: any = { ui: 'tdd', useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'), timeout: 60000 }; +// These integration tests is being run in multiple environments (electron, web, remote) +// so we need to set the suite name based on the environment as the suite name is used +// for the test results file name +let suite = ''; +if (process.env.VSCODE_BROWSER) { + suite = `${process.env.VSCODE_BROWSER} Browser Integration Notebook Tests`; +} else if (process.env.REMOTE_VSCODE) { + suite = 'Remote Integration Notebook Tests'; +} else { + suite = 'Integration Notebook Tests'; +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/test/integration/browser/src/index.ts b/test/integration/browser/src/index.ts index 91bdd3a8322..5ce67dd0548 100644 --- a/test/integration/browser/src/index.ts +++ b/test/integration/browser/src/index.ts @@ -29,7 +29,9 @@ if (optimist.argv.help) { const width = 1200; const height = 800; -async function runTestsInBrowser(browserType: 'chromium' | 'firefox' | 'webkit', endpoint: url.UrlWithStringQuery, server: cp.ChildProcess): Promise { +type BrowserType = 'chromium' | 'firefox' | 'webkit'; + +async function runTestsInBrowser(browserType: BrowserType, endpoint: url.UrlWithStringQuery, server: cp.ChildProcess): Promise { const args = process.platform === 'linux' && browserType === 'chromium' ? ['--no-sandbox'] : undefined; // disable sandbox to run chrome on certain Linux distros const browser = await playwright[browserType].launch({ headless: !Boolean(optimist.argv.debug), args }); const context = await browser.newContext(); @@ -78,7 +80,7 @@ function pkill(pid: number): Promise { }); } -async function launchServer(): Promise<{ endpoint: url.UrlWithStringQuery, server: cp.ChildProcess }> { +async function launchServer(browserType: BrowserType): Promise<{ endpoint: url.UrlWithStringQuery, server: cp.ChildProcess }> { // Ensure a tmp user-data-dir is used for the tests const tmpDir = tmp.dirSync({ prefix: 't' }); @@ -89,6 +91,7 @@ async function launchServer(): Promise<{ endpoint: url.UrlWithStringQuery, serve const env = { VSCODE_AGENT_FOLDER: userDataDir, + VSCODE_BROWSER: browserType, ...process.env }; @@ -130,7 +133,7 @@ async function launchServer(): Promise<{ endpoint: url.UrlWithStringQuery, serve }); } -launchServer().then(async ({ endpoint, server }) => { +launchServer(optimist.argv.browser).then(async ({ endpoint, server }) => { return runTestsInBrowser(optimist.argv.browser, endpoint, server); }, error => { console.error(error);