Smoke tests - run each suite in its own user-data-dir (#125670)

* Initial commit for isolated smoke tests

* Do not wait for the explorer view when restarting

* Debug failed localization test

* User data path suffix length

* Kill server when application exits

* Do not run yarn install

* Refactor application start/stop

* Reverted some changes

* Missed one

* One more change to revert

* Restored one more file

* Pull request feedback

test/automation/src/application.ts

@@ -28,6 +28,7 @@ export class Application {
 	private _workbench: Workbench | undefined;
 
 	constructor(private options: ApplicationOptions) {
+		this._userDataPath = options.userDataDir;
 		this._workspacePathOrFolder = options.workspacePath;
 	}
 
@@ -64,8 +65,9 @@ export class Application {
 		return this.options.extensionsPath;
 	}
 
+	private _userDataPath: string;
 	get userDataPath(): string {
-		return this.options.userDataDir;
+		return this._userDataPath;
 	}
 
 	async start(expectWalkthroughPart = true): Promise<any> {
@@ -123,7 +125,7 @@ export class Application {
 		this._code = await spawn({
 			codePath: this.options.codePath,
 			workspacePath: this.workspacePathOrFolder,
-			userDataDir: this.options.userDataDir,
+			userDataDir: this.userDataPath,
 			extensionsPath: this.options.extensionsPath,
 			logger: this.options.logger,
 			verbose: this.options.verbose,

test/automation/src/playwrightDriver.ts

@@ -37,7 +37,10 @@ function buildDriver(browser: playwright.Browser, page: playwright.Page): IDrive
 		},
 		capturePage: () => Promise.resolve(''),
 		reloadWindow: (windowId) => Promise.resolve(),
-		exitApplication: () => browser.close(),
+		exitApplication: async () => {
+			await browser.close();
+			await teardown();
+		},
 		dispatchKeybinding: async (windowId, keybinding) => {
 			const chords = keybinding.split(' ');
 			for (let i = 0; i < chords.length; i++) {
@@ -123,9 +126,9 @@ export async function launch(userDataDir: string, _workspacePath: string, codeSe
 	endpoint = await waitForEndpoint();
 }
 
-function teardown(): void {
+async function teardown(): Promise<void> {
 	if (server) {
-		kill(server.pid);
+		await new Promise((c, e) => kill(server!.pid, error => error ? e(error) : c(null)));
 		server = undefined;
 	}
 }

test/smoke/src/areas/editor/editor.test.ts

@@ -3,10 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Editor', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it('shows correct quick outline', async function () {
 			const app = this.app as Application;
 			await app.workbench.quickaccess.openFile('www');

test/smoke/src/areas/extensions/extensions.test.ts

@@ -3,10 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application, Quality } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Extensions', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it(`install and enable vscode-smoketest-check extension`, async function () {
 			const app = this.app as Application;
 

test/smoke/src/areas/languages/languages.test.ts

@@ -3,10 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application, ProblemSeverity, Problems } from '../../../../automation/out';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Language Features', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it('verifies quick outline', async function () {
 			const app = this.app as Application;
 			await app.workbench.quickaccess.openFile('style.css');

test/smoke/src/areas/multiroot/multiroot.test.ts

@@ -4,8 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as fs from 'fs';
+import minimist = require('minimist');
 import * as path from 'path';
 import { Application } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
 function toUri(path: string): string {
 	if (process.platform === 'win32') {
@@ -34,8 +36,9 @@ async function createWorkspaceFile(workspacePath: string): Promise<string> {
 	return workspaceFilePath;
 }
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Multiroot', () => {
+		beforeSuite(opts);
 
 		before(async function () {
 			const app = this.app as Application;
@@ -47,6 +50,8 @@ export function setup() {
 			await app.restart({ workspaceOrFolder: workspaceFilePath });
 		});
 
+		afterSuite();
+
 		it('shows results from all folders', async function () {
 			const app = this.app as Application;
 			await app.workbench.quickaccess.openQuickAccess('*.*');

test/smoke/src/areas/notebook/notebook.test.ts

@@ -4,20 +4,18 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as cp from 'child_process';
+import minimist = require('minimist');
 import { Application } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
 // function wait(ms: number): Promise<void> {
 // 	return new Promise(r => setTimeout(r, ms));
 // }
 
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Notebooks', () => {
-		after(async function () {
-			const app = this.app as Application;
-			cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder });
-			cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder });
-		});
+		beforeSuite(opts);
 
 		afterEach(async function () {
 			const app = this.app as Application;
@@ -25,6 +23,14 @@ export function setup() {
 			await app.workbench.quickaccess.runCommand('workbench.action.closeActiveEditor');
 		});
 
+		after(async function () {
+			const app = this.app as Application;
+			cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder });
+			cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder });
+		});
+
+		afterSuite();
+
 		// it('inserts/edits code cell', async function () {
 		// 	const app = this.app as Application;
 		// 	await app.workbench.notebook.openNotebook();

test/smoke/src/areas/preferences/preferences.test.ts

@@ -3,10 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application, ActivityBarPosition } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Preferences', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it('turns off editor line numbers and verifies the live change', async function () {
 			const app = this.app as Application;
 
@@ -27,13 +32,5 @@ export function setup() {
 			await app.code.dispatchKeybinding('ctrl+u');
 			await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.RIGHT);
 		});
-
-		after(async function () {
-			const app = this.app as Application;
-			await app.workbench.settingsEditor.clearUserSettings();
-
-			// Wait for settings to be applied, which will happen after the settings file is empty
-			await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.LEFT);
-		});
 	});
 }

test/smoke/src/areas/search/search.test.ts

@@ -4,17 +4,23 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as cp from 'child_process';
+import minimist = require('minimist');
 import { Application } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	// https://github.com/microsoft/vscode/issues/115244
 	describe('Search', () => {
+		beforeSuite(opts);
+
 		after(function () {
 			const app = this.app as Application;
 			cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder });
 			cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder });
 		});
 
+		afterSuite();
+
 		// https://github.com/microsoft/vscode/issues/124146
 		it.skip /* https://github.com/microsoft/vscode/issues/124335 */('has a tooltp with a keybinding', async function () {
 			const app = this.app as Application;
@@ -68,6 +74,9 @@ export function setup() {
 	});
 
 	describe('Quick Access', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it('quick access search produces correct result', async function () {
 			const app = this.app as Application;
 			const expectedNames = [

test/smoke/src/areas/statusbar/statusbar.test.ts

@@ -3,10 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application, Quality, StatusBarElement } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup(isWeb) {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Statusbar', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it('verifies presence of all default status bar elements', async function () {
 			const app = this.app as Application;
 
@@ -18,7 +23,7 @@ export function setup(isWeb) {
 			await app.workbench.statusbar.waitForStatusbarElement(StatusBarElement.PROBLEMS_STATUS);
 
 			await app.workbench.quickaccess.openFile('app.js');
-			if (!isWeb) {
+			if (!opts.web) {
 				// Encoding picker currently hidden in web (only UTF-8 supported)
 				await app.workbench.statusbar.waitForStatusbarElement(StatusBarElement.ENCODING_STATUS);
 			}
@@ -39,7 +44,7 @@ export function setup(isWeb) {
 			await app.workbench.statusbar.clickOn(StatusBarElement.INDENTATION_STATUS);
 			await app.workbench.quickinput.waitForQuickInputOpened();
 			await app.workbench.quickinput.closeQuickInput();
-			if (!isWeb) {
+			if (!opts.web) {
 				// Encoding picker currently hidden in web (only UTF-8 supported)
 				await app.workbench.statusbar.clickOn(StatusBarElement.ENCODING_STATUS);
 				await app.workbench.quickinput.waitForQuickInputOpened();

test/smoke/src/areas/workbench/data-loss.test.ts

@@ -3,10 +3,16 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
+
+export function setup(opts: minimist.ParsedArgs) {
 
-export function setup() {
 	describe('Dataloss', () => {
+		beforeSuite(opts);
+		afterSuite();
+
 		it(`verifies that 'hot exit' works for dirty files`, async function () {
 			const app = this.app as Application;
 			await app.workbench.editors.newUntitledFile();

test/smoke/src/areas/workbench/localization.test.ts

@@ -3,10 +3,14 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Application, Quality } from '../../../../automation';
+import { afterSuite, beforeSuite } from '../../utils';
 
-export function setup() {
+export function setup(opts: minimist.ParsedArgs) {
 	describe('Localization', () => {
+		beforeSuite(opts);
+
 		before(async function () {
 			const app = this.app as Application;
 
@@ -21,6 +25,8 @@ export function setup() {
 			await app.restart({ extraArgs: ['--locale=DE'] });
 		});
 
+		afterSuite();
+
 		it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () {
 			const app = this.app as Application;
 

test/smoke/src/main.ts

@@ -207,7 +207,8 @@ async function setupRepository(): Promise<void> {
 			cp.spawnSync('git', ['clean', '-xdf'], { cwd: workspacePath });
 		}
 
-		// None of the test run the project
+		// None of the current smoke tests have a dependency on the packages.
+		// If new smoke tests are added that need the packages, uncomment this.
 		// console.log('*** Running yarn...');
 		// cp.execSync('yarn', { cwd: workspacePath, stdio: 'inherit' });
 	}
@@ -299,27 +300,16 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
 	}
 
 	describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
-		before(async function () {
-			const app = new Application(this.defaultOptions);
-			await app!.start(opts.web ? false : undefined);
-			this.app = app;
-		});
-
-		after(async function () {
-			await this.app.stop();
-		});
-
-		if (!opts.web) { setupDataLossTests(); }
-		if (!opts.web) { setupDataPreferencesTests(); }
-		setupDataSearchTests();
-		setupDataNotebookTests();
-		setupDataLanguagesTests();
-		setupDataEditorTests();
-		setupDataStatusbarTests(!!opts.web);
-		setupDataExtensionTests();
-		if (!opts.web) { setupDataMultirootTests(); }
-		if (!opts.web) { setupDataLocalizationTests(); }
+		if (!opts.web) { setupDataLossTests(opts); }
+		if (!opts.web) { setupDataPreferencesTests(opts); }
+		setupDataSearchTests(opts);
+		setupDataNotebookTests(opts);
+		setupDataLanguagesTests(opts);
+		setupDataEditorTests(opts);
+		setupDataStatusbarTests(opts);
+		setupDataExtensionTests(opts);
+		if (!opts.web) { setupDataMultirootTests(opts); }
+		if (!opts.web) { setupDataLocalizationTests(opts); }
 		if (!opts.web) { setupLaunchTests(); }
 	});
 });
-

test/smoke/src/utils.ts

@@ -3,7 +3,9 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import minimist = require('minimist');
 import { Suite, Context } from 'mocha';
+import { Application, ApplicationOptions } from '../../automation';
 
 export function describeRepeat(n: number, description: string, callback: (this: Suite) => void): void {
 	for (let i = 0; i < n; i++) {
@@ -16,3 +18,25 @@ export function itRepeat(n: number, description: string, callback: (this: Contex
 		it(`${description} (iteration ${i})`, callback);
 	}
 }
+
+export function beforeSuite(opts: minimist.ParsedArgs) {
+	before(async function () {
+		// https://github.com/microsoft/vscode/issues/34988
+		const options = this.defaultOptions as ApplicationOptions;
+		const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join('');
+		const userDataDir = options.userDataDir.concat(`-${userDataPathSuffix}`);
+
+		const app = new Application({ ...options, userDataDir });
+		await app!.start(opts.web ? false : undefined);
+		this.app = app;
+	});
+}
+
+export function afterSuite() {
+	after(async function () {
+		const app = this.app as Application;
+		if (app) {
+			await app.stop();
+		}
+	});
+}