development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-10-02 17:32:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Playwright: record a trace per failing test, not suite (#138600)

Browse source 
* smoke - record traces per test and not entire suite

* smoke - only persist failing tests

* smoke - cleanup

* smoke - more logging

* smoke - push a test failure to proof the point

* smoke - switch back to chrome for smoke tests

* smoke - warn when exit takes 10s

* Revert "smoke - push a test failure to proof the point"

This reverts commit e572a0c40d.
This commit is contained in:
Benjamin Pasero 2021-12-08 10:09:30 +01:00 committed by GitHub
parent cac5f8dd96
commit fbad065eea
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 135 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/azure-pipelines/win32/product-build-win32.yml
View file

@ -229,7 +229,7 @@ steps:
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --web --browser firefox --headless }
exec { yarn smoketest-no-compile --web --headless }
displayName: Run smoke tests (Browser)
timeoutInMinutes: 10
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))

2
src/vs/platform/driver/common/driver.ts
View file

@ -41,6 +41,8 @@ export interface IDriver {
getWindowIds(): Promise<number[]>;
capturePage(windowId: number): Promise<string>;
startTracing(windowId: number, name: string): Promise<void>;
stopTracing(windowId: number, name: string, persist: boolean): Promise<void>;
reloadWindow(windowId: number): Promise<void>;
exitApplication(): Promise<boolean>;
dispatchKeybinding(windowId: number, keybinding: string): Promise<void>;

8
src/vs/platform/driver/electron-main/driver.ts
View file

@ -69,6 +69,14 @@ export class Driver implements IDriver, IWindowDriverRegistry {
return image.toPNG().toString('base64');
}
async startTracing(windowId: number, name: string): Promise<void> {
// ignore - tracing is not implemented yet
}
async stopTracing(windowId: number, name: string, persist: boolean): Promise<void> {
// ignore - tracing is not implemented yet
}
async reloadWindow(windowId: number): Promise<void> {
await this.whenUnfrozen(windowId);

10
src/vs/platform/driver/node/driver.ts
View file

@ -21,6 +21,8 @@ export class DriverChannel implements IServerChannel {
switch (command) {
case 'getWindowIds': return this.driver.getWindowIds();
case 'capturePage': return this.driver.capturePage(arg);
case 'startTracing': return this.driver.startTracing(arg[0], arg[1]);
case 'stopTracing': return this.driver.stopTracing(arg[0], arg[1], arg[2]);
case 'reloadWindow': return this.driver.reloadWindow(arg);
case 'exitApplication': return this.driver.exitApplication();
case 'dispatchKeybinding': return this.driver.dispatchKeybinding(arg[0], arg[1]);
@ -56,6 +58,14 @@ export class DriverChannelClient implements IDriver {
return this.channel.call('capturePage', windowId);
}
startTracing(windowId: number, name: string): Promise<void> {
return this.channel.call('startTracing', [windowId, name]);
}
stopTracing(windowId: number, name: string, persist: boolean): Promise<void> {
return this.channel.call('stopTracing', [windowId, name, persist]);
}
reloadWindow(windowId: number): Promise<void> {
return this.channel.call('reloadWindow', windowId);
}

8
test/automation/src/application.ts
View file

@ -104,6 +104,14 @@ export class Application {
}
}
async startTracing(name: string): Promise<void> {
await this._code?.startTracing(name);
}
async stopTracing(name: string, persist: boolean): Promise<void> {
await this._code?.stopTracing(name, persist);
}
private async startApplication(extraArgs: string[] = []): Promise<any> {
this._code = await spawn({
...this.options,

15
test/automation/src/code.ts
View file

@ -27,7 +27,6 @@ export interface SpawnOptions {
web?: boolean;
headless?: boolean;
browser?: 'chromium' | 'webkit' | 'firefox';
suiteTitle?: string;
testTitle?: string;
}
@ -134,6 +133,16 @@ export class Code {
return await this.driver.capturePage(windowId);
}
async startTracing(name: string): Promise<void> {
const windowId = await this.getActiveWindowId();
return await this.driver.startTracing(windowId, name);
}
async stopTracing(name: string, persist: boolean): Promise<void> {
const windowId = await this.getActiveWindowId();
return await this.driver.stopTracing(windowId, name, persist);
}
async waitForWindowIds(fn: (windowIds: number[]) => boolean): Promise<void> {
await poll(() => this.driver.getWindowIds(), fn, `get window ids`);
}
@ -161,6 +170,10 @@ export class Code {
while (!done) {
retries++;
if (retries > 20) {
console.warn('Smoke test exit call did not terminate process after 10s, still trying...');
}
if (retries > 40) {
done = true;
reject(new Error('Smoke test exit call did not terminate process after 20s, giving up'));

40
test/automation/src/playwrightDriver.ts
View file

@ -43,8 +43,7 @@ class PlaywrightDriver implements IDriver {
private readonly server: ChildProcess,
private readonly browser: playwright.Browser,
private readonly context: playwright.BrowserContext,
private readonly page: playwright.Page,
private readonly suiteTitle: string | undefined
private readonly page: playwright.Page
) {
}
@ -56,20 +55,34 @@ class PlaywrightDriver implements IDriver {
return '';
}
async startTracing(windowId: number, name: string): Promise<void> {
try {
await this.warnAfter(this.context.tracing.startChunk({ title: name }), 5000, 'Starting playwright trace took more than 5 seconds');
} catch (error) {
console.warn(`Failed to start playwright tracing (chunk): ${error}`);
}
}
async stopTracing(windowId: number, name: string, persist: boolean): Promise<void> {
try {
let persistPath: string | undefined = undefined;
if (persist) {
persistPath = join(logsPath, `playwright-trace-${traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
}
await this.warnAfter(this.context.tracing.stopChunk({ path: persistPath }), 5000, 'Stopping playwright trace took more than 5 seconds');
} catch (error) {
console.warn(`Failed to stop playwright tracing (chunk): ${error}`);
}
}
async reloadWindow(windowId: number) {
throw new Error('Unsupported');
}
async exitApplication() {
try {
let traceFileName: string;
if (this.suiteTitle) {
traceFileName = `playwright-trace-${traceCounter++}-${this.suiteTitle.replace(/\s+/g, '-')}.zip`;
} else {
traceFileName = `playwright-trace-${traceCounter++}.zip`;
}
await this.warnAfter(this.context.tracing.stop({ path: join(logsPath, traceFileName) }), 5000, 'Stopping playwright trace took >5seconds');
await this.warnAfter(this.context.tracing.stop(), 5000, 'Stopping playwright trace took >5seconds');
} catch (error) {
console.warn(`Failed to stop playwright tracing: ${error}`);
}
@ -196,7 +209,6 @@ let port = 9000;
export interface PlaywrightOptions {
readonly browser?: 'chromium' | 'webkit' | 'firefox';
readonly headless?: boolean;
readonly suiteTitle?: string;
}
export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, options: PlaywrightOptions = {}): Promise<{ serverProcess: ChildProcess, client: IDisposable, driver: IDriver }> {
@ -212,7 +224,7 @@ export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_P
client: {
dispose: () => { /* there is no client to dispose for browser, teardown is triggered via exitApplication call */ }
},
driver: new PlaywrightDriver(serverProcess, browser, context, page, options.suiteTitle)
driver: new PlaywrightDriver(serverProcess, browser, context, page)
};
}
@ -275,9 +287,9 @@ async function launchBrowser(options: PlaywrightOptions, endpoint: string, works
const context = await browser.newContext();
try {
await context.tracing.start({ screenshots: true, snapshots: true, sources: true, title: options.suiteTitle });
await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
} catch (error) {
console.warn(`Failed to start playwright tracing.`); // do not fail the build when this fails
console.warn(`Failed to start playwright tracing: ${error}`); // do not fail the build when this fails
}
const page = await context.newPage();

7
test/smoke/src/areas/editor/editor.test.ts
View file

@ -5,12 +5,13 @@
import minimist = require('minimist');
import { Application } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Editor', () => {
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
it('shows correct quick outline', async function () {
const app = this.app as Application;

10
test/smoke/src/areas/extensions/extensions.test.ts
View file

@ -5,14 +5,15 @@
import minimist = require('minimist');
import { Application, Quality } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Extensions', () => {
beforeSuite(opts);
afterSuite(opts);
it(`install and enable vscode-smoketest-check extension`, async function () {
// Shared before/after handling
installCommonTestHandlers(opts);
it('install and enable vscode-smoketest-check extension', async function () {
const app = this.app as Application;
if (app.quality === Quality.Dev) {
@ -29,6 +30,5 @@ export function setup(opts: minimist.ParsedArgs) {
await app.workbench.quickaccess.runCommand('Smoke Test Check');
});
});
}

7
test/smoke/src/areas/languages/languages.test.ts
View file

@ -5,12 +5,13 @@
import minimist = require('minimist');
import { Application, ProblemSeverity, Problems } from '../../../../automation/out';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Language Features', () => {
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
it('verifies quick outline', async function () {
const app = this.app as Application;

8
test/smoke/src/areas/multiroot/multiroot.test.ts
View file

@ -7,7 +7,7 @@ import * as fs from 'fs';
import minimist = require('minimist');
import * as path from 'path';
import { Application } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
function toUri(path: string): string {
if (process.platform === 'win32') {
@ -38,13 +38,13 @@ function createWorkspaceFile(workspacePath: string): string {
export function setup(opts: minimist.ParsedArgs) {
describe('Multiroot', () => {
beforeSuite(opts, async opts => {
// Shared before/after handling
installCommonTestHandlers(opts, async opts => {
const workspacePath = createWorkspaceFile(opts.workspacePath);
return { ...opts, workspacePath };
});
afterSuite(opts);
it('shows results from all folders', async function () {
const app = this.app as Application;
await app.workbench.quickaccess.openQuickAccess('*.*');

8
test/smoke/src/areas/notebook/notebook.test.ts
View file

@ -6,11 +6,13 @@
import * as cp from 'child_process';
import minimist = require('minimist');
import { Application } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe.skip('Notebooks', () => {
beforeSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
afterEach(async function () {
const app = this.app as Application;
@ -24,8 +26,6 @@ export function setup(opts: minimist.ParsedArgs) {
cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder });
});
afterSuite(opts);
it.skip('inserts/edits code cell', async function () {
const app = this.app as Application;
await app.workbench.notebook.openNotebook();

9
test/smoke/src/areas/preferences/preferences.test.ts
View file

@ -5,12 +5,13 @@
import minimist = require('minimist');
import { Application, ActivityBarPosition } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Preferences', () => {
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
it('turns off editor line numbers and verifies the live change', async function () {
const app = this.app as Application;
@ -23,7 +24,7 @@ export function setup(opts: minimist.ParsedArgs) {
await app.code.waitForElements('.line-numbers', false, result => !result || result.length === 0);
});
it(`changes 'workbench.action.toggleSidebarPosition' command key binding and verifies it`, async function () {
it('changes "workbench.action.toggleSidebarPosition" command key binding and verifies it', async function () {
const app = this.app as Application;
await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.LEFT);

13
test/smoke/src/areas/search/search.test.ts
View file

@ -6,11 +6,13 @@
import * as cp from 'child_process';
import minimist = require('minimist');
import { Application } from '../../../../automation';
import { afterSuite, beforeSuite, retry } from '../../utils';
import { installCommonTestHandlers, retry } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Search', () => {
beforeSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
after(function () {
const app = this.app as Application;
@ -18,8 +20,6 @@ export function setup(opts: minimist.ParsedArgs) {
retry(async () => cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }), 0, 5);
});
afterSuite(opts);
// https://github.com/microsoft/vscode/issues/124146
it.skip /* https://github.com/microsoft/vscode/issues/124335 */('has a tooltp with a keybinding', async function () {
const app = this.app as Application;
@ -73,8 +73,9 @@ export function setup(opts: minimist.ParsedArgs) {
});
describe('Quick Access', () => {
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
it('quick access search produces correct result', async function () {
const app = this.app as Application;

7
test/smoke/src/areas/statusbar/statusbar.test.ts
View file

@ -5,12 +5,13 @@
import minimist = require('minimist');
import { Application, Quality, StatusBarElement } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Statusbar', () => {
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
it('verifies presence of all default status bar elements', async function () {
const app = this.app as Application;

6
test/smoke/src/areas/terminal/terminal.test.ts
View file

@ -5,7 +5,7 @@
import minimist = require('minimist');
import { Application, Terminal, TerminalCommandId } from '../../../../automation/out';
import { afterSuite, beforeSuite } from '../../utils';
import { installCommonTestHandlers } from '../../utils';
import { setup as setupTerminalEditorsTests } from './terminal-editors.test';
import { setup as setupTerminalPersistenceTests } from './terminal-persistence.test';
import { setup as setupTerminalProfileTests } from './terminal-profiles.test';
@ -21,8 +21,8 @@ export function setup(opts: minimist.ParsedArgs) {
// Retry tests 3 times to minimize build failures due to any flakiness
this.retries(3);
beforeSuite(opts);
afterSuite(opts);
// Shared before/after handling
installCommonTestHandlers(opts);
let terminal: Terminal;
before(async function () {

7
test/smoke/src/areas/workbench/data-loss.test.ts
View file

@ -5,15 +5,14 @@
import { Application, ApplicationOptions, Quality } from '../../../../automation/out';
import { ParsedArgs } from 'minimist';
import { afterSuite, getRandomUserDataDir, startApp, timeout } from '../../utils';
import { installCommonAfterHandlers, getRandomUserDataDir, startApp, timeout } from '../../utils';
export function setup(opts: ParsedArgs) {
describe('Data Loss (insiders -> insiders)', () => {
let app: Application | undefined = undefined;
afterSuite(opts, () => app);
installCommonAfterHandlers(opts, () => app);
it('verifies opened editors are restored', async function () {
app = await startApp(opts, this.defaultOptions);
@ -98,7 +97,7 @@ export function setup(opts: ParsedArgs) {
let insidersApp: Application | undefined = undefined;
let stableApp: Application | undefined = undefined;
afterSuite(opts, () => insidersApp ?? stableApp, async () => stableApp?.stop());
installCommonAfterHandlers(opts, () => insidersApp ?? stableApp, async () => stableApp?.stop());
it('verifies opened editors are restored', async function () {
const stableCodePath = opts['stable-build'];

5
test/smoke/src/areas/workbench/launch.test.ts
View file

@ -6,15 +6,14 @@
import minimist = require('minimist');
import { join } from 'path';
import { Application } from '../../../../automation';
import { afterSuite, startApp } from '../../utils';
import { installCommonAfterHandlers, startApp } from '../../utils';
export function setup(args: minimist.ParsedArgs) {
describe('Launch', () => {
let app: Application | undefined;
afterSuite(args, () => app);
installCommonAfterHandlers(args, () => app);
it(`verifies that application launches when user data directory has non-ascii characters`, async function () {
const massagedOptions = { ...this.defaultOptions, userDataDir: join(this.defaultOptions.userDataDir, 'ø') };

4
test/smoke/src/areas/workbench/localization.test.ts
View file

@ -5,7 +5,7 @@
import minimist = require('minimist');
import { Application, Quality } from '../../../../automation';
import { afterSuite, startApp } from '../../utils';
import { installCommonAfterHandlers, startApp } from '../../utils';
export function setup(args: minimist.ParsedArgs) {
@ -13,7 +13,7 @@ export function setup(args: minimist.ParsedArgs) {
let app: Application | undefined = undefined;
afterSuite(args, () => app);
installCommonAfterHandlers(args, () => app);
it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () {
if (this.defaultOptions.quality === Quality.Dev || this.defaultOptions.remote) {

23
test/smoke/src/utils.ts
View file

@ -19,13 +19,16 @@ export function itRepeat(n: number, description: string, callback: (this: Contex
}
}
export function beforeSuite(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>) {
export function installCommonTestHandlers(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>) {
installCommonBeforeHandlers(args, optionsTransform);
installCommonAfterHandlers(args);
}
export function installCommonBeforeHandlers(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>) {
before(async function () {
const testTitle = this.currentTest?.title;
const suiteTitle = this.currentTest?.parent?.title;
this.app = await startApp(args, this.defaultOptions, async opts => {
opts.suiteTitle = suiteTitle;
opts.testTitle = testTitle;
if (optionsTransform) {
@ -35,6 +38,12 @@ export function beforeSuite(args: minimist.ParsedArgs, optionsTransform?: (opts:
return opts;
});
});
beforeEach(async function () {
if (this.app) {
await this.app.startTracing(this.currentTest?.title);
}
});
}
export async function startApp(args: minimist.ParsedArgs, options: ApplicationOptions, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>): Promise<Application> {
@ -66,7 +75,7 @@ export function getRandomUserDataDir(options: ApplicationOptions): string {
return options.userDataDir.concat(`-${userDataPathSuffix}`);
}
export function afterSuite(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise<unknown>) {
export function installCommonAfterHandlers(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise<unknown>) {
after(async function () {
const app: Application = appFn?.() ?? this.app;
@ -87,6 +96,12 @@ export function afterSuite(opts: minimist.ParsedArgs, appFn?: () => Application
await joinFn();
}
});
afterEach(async function () {
if (this.app) {
await this.app.stopTracing(this.currentTest?.title, this.currentTest?.state === 'failed');
}
});
}
export function timeout(i: number) {