From 3c4fdb58a18d17a04e7e49c017461e0d058ea296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moreno?= Date: Fri, 16 Jul 2021 11:58:49 +0200 Subject: [PATCH] fix screenshots in smoketests related to #122685 --- test/smoke/src/areas/editor/editor.test.ts | 4 +- .../src/areas/extensions/extensions.test.ts | 2 +- .../src/areas/languages/languages.test.ts | 2 +- .../src/areas/multiroot/multiroot.test.ts | 2 +- .../smoke/src/areas/notebook/notebook.test.ts | 2 +- .../src/areas/preferences/preferences.test.ts | 2 +- test/smoke/src/areas/search/search.test.ts | 4 +- .../src/areas/statusbar/statusbar.test.ts | 2 +- .../src/areas/workbench/data-loss.test.ts | 2 +- .../src/areas/workbench/localization.test.ts | 2 +- test/smoke/src/main.ts | 67 +++++-------------- test/smoke/src/utils.ts | 13 +++- 12 files changed, 42 insertions(+), 62 deletions(-) diff --git a/test/smoke/src/areas/editor/editor.test.ts b/test/smoke/src/areas/editor/editor.test.ts index 1d3b1feddf9..9fb676ebf69 100644 --- a/test/smoke/src/areas/editor/editor.test.ts +++ b/test/smoke/src/areas/editor/editor.test.ts @@ -10,14 +10,14 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Editor', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it('shows correct quick outline', async function () { const app = this.app as Application; await app.workbench.quickaccess.openFile('www'); await app.workbench.quickaccess.openQuickOutline(); - await app.workbench.quickinput.waitForQuickInputElements(names => names.length >= 6); + await app.workbench.quickinput.waitForQuickInputElements(names => names.length >= 6000); }); }); } diff --git a/test/smoke/src/areas/extensions/extensions.test.ts b/test/smoke/src/areas/extensions/extensions.test.ts index 875f41b152a..5ee13542b00 100644 --- a/test/smoke/src/areas/extensions/extensions.test.ts +++ b/test/smoke/src/areas/extensions/extensions.test.ts @@ -10,7 +10,7 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Extensions', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it(`install and enable vscode-smoketest-check extension`, async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/languages/languages.test.ts b/test/smoke/src/areas/languages/languages.test.ts index 3112c08c3e1..b4d6f67f130 100644 --- a/test/smoke/src/areas/languages/languages.test.ts +++ b/test/smoke/src/areas/languages/languages.test.ts @@ -10,7 +10,7 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Language Features', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it('verifies quick outline', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/multiroot/multiroot.test.ts b/test/smoke/src/areas/multiroot/multiroot.test.ts index 9fa052498e8..94350d9a488 100644 --- a/test/smoke/src/areas/multiroot/multiroot.test.ts +++ b/test/smoke/src/areas/multiroot/multiroot.test.ts @@ -43,7 +43,7 @@ export function setup(opts: minimist.ParsedArgs) { return { ...opts, workspacePath }; }); - afterSuite(); + afterSuite(opts); it('shows results from all folders', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/notebook/notebook.test.ts b/test/smoke/src/areas/notebook/notebook.test.ts index 866e4154902..55a471578b4 100644 --- a/test/smoke/src/areas/notebook/notebook.test.ts +++ b/test/smoke/src/areas/notebook/notebook.test.ts @@ -24,7 +24,7 @@ export function setup(opts: minimist.ParsedArgs) { cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); }); - afterSuite(); + afterSuite(opts); it.skip('inserts/edits code cell', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/preferences/preferences.test.ts b/test/smoke/src/areas/preferences/preferences.test.ts index 1138e7da0ba..af363b08661 100644 --- a/test/smoke/src/areas/preferences/preferences.test.ts +++ b/test/smoke/src/areas/preferences/preferences.test.ts @@ -10,7 +10,7 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Preferences', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it('turns off editor line numbers and verifies the live change', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/search/search.test.ts b/test/smoke/src/areas/search/search.test.ts index 274d96d7cb3..5bc1b4494d7 100644 --- a/test/smoke/src/areas/search/search.test.ts +++ b/test/smoke/src/areas/search/search.test.ts @@ -19,7 +19,7 @@ export function setup(opts: minimist.ParsedArgs) { cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); }); - afterSuite(); + afterSuite(opts); // https://github.com/microsoft/vscode/issues/124146 it.skip /* https://github.com/microsoft/vscode/issues/124335 */('has a tooltp with a keybinding', async function () { @@ -75,7 +75,7 @@ export function setup(opts: minimist.ParsedArgs) { describe('Quick Access', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it('quick access search produces correct result', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/statusbar/statusbar.test.ts b/test/smoke/src/areas/statusbar/statusbar.test.ts index b0744f0a906..81d6f335e10 100644 --- a/test/smoke/src/areas/statusbar/statusbar.test.ts +++ b/test/smoke/src/areas/statusbar/statusbar.test.ts @@ -10,7 +10,7 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Statusbar', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it('verifies presence of all default status bar elements', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/workbench/data-loss.test.ts b/test/smoke/src/areas/workbench/data-loss.test.ts index 51dda345fc8..4abc0d1c7ec 100644 --- a/test/smoke/src/areas/workbench/data-loss.test.ts +++ b/test/smoke/src/areas/workbench/data-loss.test.ts @@ -11,7 +11,7 @@ export function setup(opts: minimist.ParsedArgs) { describe('Dataloss', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it(`verifies that 'hot exit' works for dirty files`, async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/workbench/localization.test.ts b/test/smoke/src/areas/workbench/localization.test.ts index bacd3d7396d..15aeba0ff6b 100644 --- a/test/smoke/src/areas/workbench/localization.test.ts +++ b/test/smoke/src/areas/workbench/localization.test.ts @@ -10,7 +10,7 @@ import { afterSuite, beforeSuite } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Localization', () => { beforeSuite(opts); - afterSuite(); + afterSuite(opts); it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () { const app = this.app as Application; diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 17122480168..2c7d9dc6483 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -13,15 +13,7 @@ import * as mkdirp from 'mkdirp'; import { ncp } from 'ncp'; import * as vscodetest from 'vscode-test'; import fetch from 'node-fetch'; -import { - Application, - Quality, - ApplicationOptions, - MultiLogger, - Logger, - ConsoleLogger, - FileLogger, -} from '../../automation'; +import { Quality, ApplicationOptions, MultiLogger, Logger, ConsoleLogger, FileLogger } from '../../automation'; import { setup as setupDataMigrationTests } from './areas/workbench/data-migration.test'; import { setup as setupDataLossTests } from './areas/workbench/data-loss.test'; @@ -335,45 +327,22 @@ after(async function () { await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c(undefined))); }); -describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { - if (screenshotsPath) { - afterEach(async function () { - if (this.currentTest!.state !== 'failed') { - return; - } - const app = this.app as Application; - const name = this.currentTest!.fullTitle().replace(/[^a-z0-9\-]/ig, '_'); - - await app.captureScreenshot(name); - }); - } - - if (opts.log) { - beforeEach(async function () { - const app = this.app as Application; - const title = this.currentTest!.fullTitle(); - - app.logger.log('*** Test start:', title); - }); - } - - if (!opts.web && opts['build'] && !opts['remote']) { - describe(`Stable vs Insiders Smoke Tests: This test MUST run before releasing`, () => { - setupDataMigrationTests(opts, testDataPath); - }); - } - - describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { - if (!opts.web) { setupDataLossTests(opts); } - if (!opts.web) { setupDataPreferencesTests(opts); } - setupDataSearchTests(opts); - setupDataNotebookTests(opts); - setupDataLanguagesTests(opts); - setupDataEditorTests(opts); - setupDataStatusbarTests(opts); - setupDataExtensionTests(opts); - if (!opts.web) { setupDataMultirootTests(opts); } - if (!opts.web) { setupDataLocalizationTests(opts); } - if (!opts.web) { setupLaunchTests(); } +if (!opts.web && opts['build'] && !opts['remote']) { + describe(`Stable vs Insiders Smoke Tests: This test MUST run before releasing`, () => { + setupDataMigrationTests(opts, testDataPath); }); +} + +describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { + if (!opts.web) { setupDataLossTests(opts); } + if (!opts.web) { setupDataPreferencesTests(opts); } + setupDataSearchTests(opts); + setupDataNotebookTests(opts); + setupDataLanguagesTests(opts); + setupDataEditorTests(opts); + setupDataStatusbarTests(opts); + setupDataExtensionTests(opts); + if (!opts.web) { setupDataMultirootTests(opts); } + if (!opts.web) { setupDataLocalizationTests(opts); } + if (!opts.web) { setupLaunchTests(); } }); diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index d0141fcd104..d79ad3388d1 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -34,12 +34,23 @@ export function beforeSuite(opts: minimist.ParsedArgs, optionsTransform?: (opts: const app = new Application({ ...options, userDataDir }); await app!.start(opts.web ? false : undefined); this.app = app; + + if (opts.log) { + const title = this.currentTest!.fullTitle(); + app.logger.log('*** Test start:', title); + } }); } -export function afterSuite() { +export function afterSuite(opts: minimist.ParsedArgs) { after(async function () { const app = this.app as Application; + + if (this.currentTest?.state === 'failed' && opts.screenshots) { + const name = this.currentTest!.fullTitle().replace(/[^a-z0-9\-]/ig, '_'); + await app.captureScreenshot(name); + } + if (app) { await app.stop(); }