Enable automated smoketests on Windows (#128224)

build/azure-pipelines/win32/product-build-win32.yml

@@ -17,7 +17,7 @@ steps:
     inputs:
       azureSubscription: "vscode-builds-subscription"
       KeyVaultName: vscode
-      SecretsFilter: 'github-distro-mixin-password,ESRP-SSL-AADAuth,vscode-storage-key,builds-docdb-key-readwrite'
+      SecretsFilter: "github-distro-mixin-password,ESRP-SSL-AADAuth,vscode-storage-key,builds-docdb-key-readwrite"
 
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -189,6 +189,41 @@ steps:
     timeoutInMinutes: 7
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { yarn --cwd test/smoke compile }
+    displayName: Compile smoke tests
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --build "$AppRoot" }
+    displayName: Run smoke tests (Electron)
+    timeoutInMinutes: 5
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  # - powershell: |
+  #     . build/azure-pipelines/win32/exec.ps1
+  #     $ErrorActionPreference = "Stop"
+  #     $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+  #     $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
+  #     exec { yarn smoketest-no-compile --build "$AppRoot" --remote }
+  #   displayName: Run smoke tests (Remote)
+  #   timeoutInMinutes: 5
+  #   condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --web --browser firefox --headless }
+    displayName: Run smoke tests (Browser)
+    timeoutInMinutes: 5
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
   - task: PublishPipelineArtifact@0
     inputs:
       artifactName: crash-dump-windows-$(VSCODE_ARCH)

test/automation/src/code.ts

@@ -97,11 +97,9 @@ export interface SpawnOptions {
 	verbose?: boolean;
 	extraArgs?: string[];
 	log?: string;
-	/** Run in the test resolver */
 	remote?: boolean;
-	/** Run in the web */
 	web?: boolean;
-	/** A specific browser to use (requires web: true) */
+	headless?: boolean;
 	browser?: 'chromium' | 'webkit' | 'firefox';
 }
 
@@ -124,7 +122,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
 
 	if (options.web) {
 		await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath, Boolean(options.verbose));
-		connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, options.browser);
+		connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, options);
 		return connect(connectDriver, child, '', handle, options.logger);
 	}
 
@@ -174,6 +172,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
 		env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
 	}
 
+	const spawnOptions: cp.SpawnOptions = { env };
 
 	args.push('--enable-proposed-api=vscode.vscode-notebook-tests');
 
@@ -183,6 +182,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
 
 	if (options.verbose) {
 		args.push('--driver-verbose');
+		spawnOptions.stdio = ['ignore', 'inherit', 'inherit'];
 	}
 
 	if (options.log) {
@@ -194,7 +194,6 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
 	}
 
 	const electronPath = codePath ? getBuildElectronPath(codePath) : getDevElectronPath();
-	const spawnOptions: cp.SpawnOptions = { env };
 	child = cp.spawn(electronPath, args, spawnOptions);
 	instances.add(child);
 	child.once('exit', () => instances.delete(child!));

test/automation/src/playwrightDriver.ts

@@ -155,7 +155,12 @@ export async function launch(userDataDir: string, _workspacePath: string, codeSe
 
 async function teardown(): Promise<void> {
 	if (server) {
-		await new Promise((c, e) => kill(server!.pid, error => error ? e(error) : c(null)));
+		try {
+			await new Promise<void>((c, e) => kill(server!.pid, err => err ? e(err) : c()));
+		} catch {
+			// noop
+		}
+
 		server = undefined;
 	}
 }
@@ -171,9 +176,14 @@ function waitForEndpoint(): Promise<string> {
 	});
 }
 
-export function connect(browserType: 'chromium' | 'webkit' | 'firefox' = 'chromium'): Promise<{ client: IDisposable, driver: IDriver }> {
+interface Options {
+	readonly browser?: 'chromium' | 'webkit' | 'firefox';
+	readonly headless?: boolean;
+}
+
+export function connect(options: Options = {}): Promise<{ client: IDisposable, driver: IDriver }> {
 	return new Promise(async (c) => {
-		const browser = await playwright[browserType].launch({ headless: false });
+		const browser = await playwright[options.browser ?? 'chromium'].launch({ headless: options.headless ?? false });
 		const context = await browser.newContext();
 		await context.tracing.start({ screenshots: true, snapshots: true });
 		const page = await context.newPage();

test/smoke/src/main.ts

@@ -59,7 +59,8 @@ const opts = minimist(args, {
 	boolean: [
 		'verbose',
 		'remote',
-		'web'
+		'web',
+		'headless'
 	],
 	default: {
 		verbose: false
@@ -257,6 +258,7 @@ function createOptions(): ApplicationOptions {
 		screenshotsPath,
 		remote: opts.remote,
 		web: opts.web,
+		headless: opts.headless,
 		browser: opts.browser,
 		extraArgs: (opts.electronArgs || '').split(' ').map(a => a.trim()).filter(a => !!a)
 	};