mirror of
https://github.com/Microsoft/vscode
synced 2024-10-02 09:18:59 +00:00
smoke - reimplement logging
This commit is contained in:
Benjamin Pasero
parent
18e8272604
commit
64562e2a0e
|
|
@ -77,8 +77,9 @@ export class Application {
|
|||
|
||||
private async _start(workspaceOrFolder = this.workspacePathOrFolder, extraArgs: string[] = []): Promise<any> {
|
||||
this._workspacePathOrFolder = workspaceOrFolder;
|
||||
await this.startApplication(extraArgs);
|
||||
await this.checkWindowReady();
|
||||
|
||||
const code = await this.startApplication(extraArgs);
|
||||
await this.checkWindowReady(code);
|
||||
}
|
||||
|
||||
async stop(): Promise<any> {
|
||||
|
|
@ -96,9 +97,7 @@ export class Application {
|
|||
const raw = await this.code.capturePage();
|
||||
const buffer = Buffer.from(raw, 'base64');
|
||||
const screenshotPath = path.join(this.options.screenshotsPath, `${name}.png`);
|
||||
if (this.options.log) {
|
||||
this.logger.log('*** Screenshot recorded:', screenshotPath);
|
||||
}
|
||||
this.logger.log('Screenshot recorded:', screenshotPath);
|
||||
|
||||
fs.writeFileSync(screenshotPath, buffer);
|
||||
}
|
||||
|
|
@ -112,26 +111,23 @@ export class Application {
|
|||
await this._code?.stopTracing(name, persist);
|
||||
}
|
||||
|
||||
private async startApplication(extraArgs: string[] = []): Promise<any> {
|
||||
this._code = await spawn({
|
||||
private async startApplication(extraArgs: string[] = []): Promise<Code> {
|
||||
const code = this._code = await spawn({
|
||||
...this.options,
|
||||
extraArgs: [...(this.options.extraArgs || []), ...extraArgs],
|
||||
});
|
||||
|
||||
this._workbench = new Workbench(this._code, this.userDataPath);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
private async checkWindowReady(): Promise<any> {
|
||||
if (!this.code) {
|
||||
console.error('No code instance found');
|
||||
return;
|
||||
}
|
||||
|
||||
await this.code.waitForWindowIds(ids => ids.length > 0);
|
||||
await this.code.waitForElement('.monaco-workbench');
|
||||
private async checkWindowReady(code: Code): Promise<any> {
|
||||
await code.waitForWindowIds(ids => ids.length > 0);
|
||||
await code.waitForElement('.monaco-workbench');
|
||||
|
||||
if (this.remote) {
|
||||
await this.code.waitForTextContent('.monaco-workbench .statusbar-item[id="status.host"]', ' TestResolver', undefined, 2000);
|
||||
await code.waitForTextContent('.monaco-workbench .statusbar-item[id="status.host"]', ' TestResolver', undefined, 2000);
|
||||
}
|
||||
|
||||
// wait a bit, since focus might be stolen off widgets
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ export interface SpawnOptions {
|
|||
logger: Logger;
|
||||
verbose?: boolean;
|
||||
extraArgs?: string[];
|
||||
log?: string;
|
||||
remote?: boolean;
|
||||
web?: boolean;
|
||||
headless?: boolean;
|
||||
|
|
@ -52,13 +51,13 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
|
|||
}
|
||||
|
||||
async function spawnBrowser(options: SpawnOptions): Promise<Code> {
|
||||
const { serverProcess, client, driver } = await launchPlaywright(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), options);
|
||||
const { serverProcess, client, driver } = await launchPlaywright(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), options, options.logger);
|
||||
|
||||
return new Code(client, driver, options.logger, serverProcess);
|
||||
}
|
||||
|
||||
async function spawnElectron(options: SpawnOptions): Promise<Code> {
|
||||
const { electronProcess, client, driver } = await launchElectron(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), Boolean(options.remote), options.log, options.extraArgs);
|
||||
const { electronProcess, client, driver } = await launchElectron(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), Boolean(options.remote), options.extraArgs, options.logger);
|
||||
|
||||
return new Code(client, driver, options.logger, electronProcess);
|
||||
}
|
||||
|
|
@ -66,6 +65,7 @@ async function spawnElectron(options: SpawnOptions): Promise<Code> {
|
|||
async function poll<T>(
|
||||
fn: () => Thenable<T>,
|
||||
acceptFn: (result: T) => boolean,
|
||||
logger: Logger,
|
||||
timeoutMessage: string,
|
||||
retryCount: number = 200,
|
||||
retryInterval: number = 100 // millis
|
||||
|
|
@ -75,9 +75,9 @@ async function poll<T>(
|
|||
|
||||
while (true) {
|
||||
if (trial > retryCount) {
|
||||
console.error('** Timeout!');
|
||||
console.error(lastError);
|
||||
console.error(`*** Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
|
||||
logger.log('Timeout!');
|
||||
logger.log(lastError);
|
||||
logger.log(`Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
|
||||
throw new Error(`Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
|
||||
}
|
||||
|
||||
|
|
@ -144,7 +144,7 @@ export class Code {
|
|||
}
|
||||
|
||||
async waitForWindowIds(fn: (windowIds: number[]) => boolean): Promise<void> {
|
||||
await poll(() => this.driver.getWindowIds(), fn, `get window ids`);
|
||||
await poll(() => this.driver.getWindowIds(), fn, this.logger, `get window ids`);
|
||||
}
|
||||
|
||||
async dispatchKeybinding(keybinding: string): Promise<void> {
|
||||
|
|
@ -171,7 +171,7 @@ export class Code {
|
|||
retries++;
|
||||
|
||||
if (retries > 20) {
|
||||
console.warn('Smoke test exit call did not terminate process after 10s, still trying...');
|
||||
this.logger.log('Smoke test exit call did not terminate process after 10s, still trying...');
|
||||
}
|
||||
|
||||
if (retries > 40) {
|
||||
|
|
@ -200,6 +200,7 @@ export class Code {
|
|||
return await poll(
|
||||
() => this.driver.getElements(windowId, selector).then(els => els.length > 0 ? Promise.resolve(els[0].textContent) : Promise.reject(new Error('Element not found for textContent'))),
|
||||
s => accept!(typeof s === 'string' ? s : ''),
|
||||
this.logger,
|
||||
`get text content '${selector}'`,
|
||||
retryCount
|
||||
);
|
||||
|
|
@ -207,52 +208,52 @@ export class Code {
|
|||
|
||||
async waitAndClick(selector: string, xoffset?: number, yoffset?: number, retryCount: number = 200): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.click(windowId, selector, xoffset, yoffset), () => true, `click '${selector}'`, retryCount);
|
||||
await poll(() => this.driver.click(windowId, selector, xoffset, yoffset), () => true, this.logger, `click '${selector}'`, retryCount);
|
||||
}
|
||||
|
||||
async waitAndDoubleClick(selector: string): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.doubleClick(windowId, selector), () => true, `double click '${selector}'`);
|
||||
await poll(() => this.driver.doubleClick(windowId, selector), () => true, this.logger, `double click '${selector}'`);
|
||||
}
|
||||
|
||||
async waitForSetValue(selector: string, value: string): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.setValue(windowId, selector, value), () => true, `set value '${selector}'`);
|
||||
await poll(() => this.driver.setValue(windowId, selector, value), () => true, this.logger, `set value '${selector}'`);
|
||||
}
|
||||
|
||||
async waitForElements(selector: string, recursive: boolean, accept: (result: IElement[]) => boolean = result => result.length > 0): Promise<IElement[]> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
return await poll(() => this.driver.getElements(windowId, selector, recursive), accept, `get elements '${selector}'`);
|
||||
return await poll(() => this.driver.getElements(windowId, selector, recursive), accept, this.logger, `get elements '${selector}'`);
|
||||
}
|
||||
|
||||
async waitForElement(selector: string, accept: (result: IElement | undefined) => boolean = result => !!result, retryCount: number = 200): Promise<IElement> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
return await poll<IElement>(() => this.driver.getElements(windowId, selector).then(els => els[0]), accept, `get element '${selector}'`, retryCount);
|
||||
return await poll<IElement>(() => this.driver.getElements(windowId, selector).then(els => els[0]), accept, this.logger, `get element '${selector}'`, retryCount);
|
||||
}
|
||||
|
||||
async waitForActiveElement(selector: string, retryCount: number = 200): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.isActiveElement(windowId, selector), r => r, `is active element '${selector}'`, retryCount);
|
||||
await poll(() => this.driver.isActiveElement(windowId, selector), r => r, this.logger, `is active element '${selector}'`, retryCount);
|
||||
}
|
||||
|
||||
async waitForTitle(fn: (title: string) => boolean): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.getTitle(windowId), fn, `get title`);
|
||||
await poll(() => this.driver.getTitle(windowId), fn, this.logger, `get title`);
|
||||
}
|
||||
|
||||
async waitForTypeInEditor(selector: string, text: string): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.typeInEditor(windowId, selector, text), () => true, `type in editor '${selector}'`);
|
||||
await poll(() => this.driver.typeInEditor(windowId, selector, text), () => true, this.logger, `type in editor '${selector}'`);
|
||||
}
|
||||
|
||||
async waitForTerminalBuffer(selector: string, accept: (result: string[]) => boolean): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.getTerminalBuffer(windowId, selector), accept, `get terminal buffer '${selector}'`);
|
||||
await poll(() => this.driver.getTerminalBuffer(windowId, selector), accept, this.logger, `get terminal buffer '${selector}'`);
|
||||
}
|
||||
|
||||
async writeInTerminal(selector: string, value: string): Promise<void> {
|
||||
const windowId = await this.getActiveWindowId();
|
||||
await poll(() => this.driver.writeInTerminal(windowId, selector, value), () => true, `writeInTerminal '${selector}'`);
|
||||
await poll(() => this.driver.writeInTerminal(windowId, selector, value), () => true, this.logger, `writeInTerminal '${selector}'`);
|
||||
}
|
||||
|
||||
async getLocaleInfo(): Promise<ILocaleInfo> {
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ import { promisify } from 'util';
|
|||
import * as kill from 'tree-kill';
|
||||
import { copyExtension } from './extensions';
|
||||
import { URI } from 'vscode-uri';
|
||||
import { Logger } from './logger';
|
||||
|
||||
const repoPath = path.join(__dirname, '../../..');
|
||||
|
||||
export async function launch(codePath: string | undefined, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, remote: boolean, log: string | undefined, extraArgs: string[] | undefined): Promise<{ electronProcess: ChildProcess, client: IDisposable, driver: IDriver }> {
|
||||
export async function launch(codePath: string | undefined, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, remote: boolean, extraArgs: string[] | undefined, logger: Logger): Promise<{ electronProcess: ChildProcess, client: IDisposable, driver: IDriver }> {
|
||||
const env = { ...process.env };
|
||||
const logsPath = path.join(repoPath, '.build', 'logs', remote ? 'smoke-tests-remote' : 'smoke-tests');
|
||||
const outPath = codePath ? getBuildOutPath(codePath) : getDevOutPath();
|
||||
|
|
@ -79,10 +80,6 @@ export async function launch(codePath: string | undefined, userDataDir: string,
|
|||
spawnOptions.stdio = ['ignore', 'inherit', 'inherit'];
|
||||
}
|
||||
|
||||
if (log) {
|
||||
args.push('--log', log);
|
||||
}
|
||||
|
||||
if (extraArgs) {
|
||||
args.push(...extraArgs);
|
||||
}
|
||||
|
|
@ -91,13 +88,13 @@ export async function launch(codePath: string | undefined, userDataDir: string,
|
|||
const electronProcess = spawn(electronPath, args, spawnOptions);
|
||||
|
||||
if (verbose) {
|
||||
console.info(`*** Started electron for desktop smoke tests on pid ${electronProcess.pid}`);
|
||||
logger.log(`Started electron for desktop smoke tests on pid ${electronProcess.pid}`);
|
||||
}
|
||||
|
||||
let electronProcessDidExit = false;
|
||||
electronProcess.once('exit', (code, signal) => {
|
||||
if (verbose) {
|
||||
console.info(`*** Electron for desktop smoke tests terminated (pid: ${electronProcess.pid}, code: ${code}, signal: ${signal})`);
|
||||
logger.log(`Electron for desktop smoke tests terminated (pid: ${electronProcess.pid}, code: ${code}, signal: ${signal})`);
|
||||
}
|
||||
electronProcessDidExit = true;
|
||||
});
|
||||
|
|
@ -118,12 +115,12 @@ export async function launch(codePath: string | undefined, userDataDir: string,
|
|||
|
||||
// give up
|
||||
if (++retries > 30) {
|
||||
console.error(`*** Error connecting driver: ${err}. Giving up...`);
|
||||
logger.log(`Error connecting driver: ${err}. Giving up...`);
|
||||
|
||||
try {
|
||||
await promisify(kill)(electronProcess.pid!);
|
||||
} catch (error) {
|
||||
console.warn(`*** Error tearing down electron client (pid: ${electronProcess.pid}): ${error}`);
|
||||
logger.log(`Error tearing down electron client (pid: ${electronProcess.pid}): ${error}`);
|
||||
}
|
||||
|
||||
throw err;
|
||||
|
|
@ -132,7 +129,7 @@ export async function launch(codePath: string | undefined, userDataDir: string,
|
|||
// retry
|
||||
else {
|
||||
if ((err as NodeJS.ErrnoException).code !== 'ENOENT' /* ENOENT is expected for as long as the server has not started on the socket */) {
|
||||
console.error(`*** Error connecting driver: ${err}. Attempting to retry...`);
|
||||
logger.log(`Error connecting driver: ${err}. Attempting to retry...`);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
|
|
|||
|
|
@ -39,4 +39,27 @@ export class MultiLogger implements Logger {
|
|||
logger.log(message, ...args);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function measureAndLog<T>(promise: Promise<T>, name: string, logger: Logger): Promise<T> {
|
||||
const now = Date.now();
|
||||
|
||||
logger.log(`Starting operation '${name}...`);
|
||||
|
||||
let res: T | undefined = undefined;
|
||||
let e: unknown;
|
||||
try {
|
||||
res = await promise;
|
||||
} catch (error) {
|
||||
e = error;
|
||||
}
|
||||
|
||||
if (e) {
|
||||
logger.log(`Finished operation '${name}' with error ${e} after ${Date.now() - now}ms`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
logger.log(`Finished operation '${name}' successfully after ${Date.now() - now}ms`);
|
||||
|
||||
return res as unknown as T;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import { IDriver, IDisposable, IWindowDriver } from './driver';
|
|||
import { URI } from 'vscode-uri';
|
||||
import * as kill from 'tree-kill';
|
||||
import { PageFunction } from 'playwright-core/types/structs';
|
||||
import { Logger } from './logger';
|
||||
import { measureAndLog } from '.';
|
||||
|
||||
const width = 1200;
|
||||
const height = 800;
|
||||
|
|
@ -43,7 +45,8 @@ class PlaywrightDriver implements IDriver {
|
|||
private readonly server: ChildProcess,
|
||||
private readonly browser: playwright.Browser,
|
||||
private readonly context: playwright.BrowserContext,
|
||||
private readonly page: playwright.Page
|
||||
private readonly page: playwright.Page,
|
||||
private readonly logger: Logger
|
||||
) {
|
||||
}
|
||||
|
||||
|
|
@ -57,9 +60,9 @@ class PlaywrightDriver implements IDriver {
|
|||
|
||||
async startTracing(windowId: number, name: string): Promise<void> {
|
||||
try {
|
||||
await this.warnAfter(this.context.tracing.startChunk({ title: name }), 5000, 'Starting playwright trace took more than 5 seconds');
|
||||
await measureAndLog(this.context.tracing.startChunk({ title: name }), `startTracing for ${name}`, this.logger);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to start playwright tracing (chunk): ${error}`);
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -70,9 +73,9 @@ class PlaywrightDriver implements IDriver {
|
|||
persistPath = join(logsPath, `playwright-trace-${traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
|
||||
}
|
||||
|
||||
await this.warnAfter(this.context.tracing.stopChunk({ path: persistPath }), 5000, 'Stopping playwright trace took more than 5 seconds');
|
||||
await measureAndLog(this.context.tracing.stopChunk({ path: persistPath }), `stopTracing for ${name}`, this.logger);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to stop playwright tracing (chunk): ${error}`);
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -82,32 +85,22 @@ class PlaywrightDriver implements IDriver {
|
|||
|
||||
async exitApplication() {
|
||||
try {
|
||||
await this.warnAfter(this.context.tracing.stop(), 5000, 'Stopping playwright trace took >5seconds');
|
||||
await measureAndLog(this.context.tracing.stop(), 'stop tracing', this.logger);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to stop playwright tracing: ${error}`);
|
||||
// Ignore
|
||||
}
|
||||
|
||||
try {
|
||||
await this.warnAfter(this.browser.close(), 5000, 'Closing playwright browser took >5seconds');
|
||||
await measureAndLog(this.browser.close(), 'Browser.close()', this.logger);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to close browser: ${error}`);
|
||||
// Ignore
|
||||
}
|
||||
|
||||
await this.warnAfter(teardown(this.server), 5000, 'Tearing down server took >5seconds');
|
||||
await measureAndLog(teardown(this.server, this.logger), 'teardown server', this.logger);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async warnAfter(promise: Promise<void>, delay: number, msg: string): Promise<void> {
|
||||
const timeout = setTimeout(() => console.warn(msg), delay);
|
||||
|
||||
try {
|
||||
await promise;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchKeybinding(windowId: number, keybinding: string) {
|
||||
const chords = keybinding.split(' ');
|
||||
for (let i = 0; i < chords.length; i++) {
|
||||
|
|
@ -211,24 +204,24 @@ export interface PlaywrightOptions {
|
|||
readonly headless?: boolean;
|
||||
}
|
||||
|
||||
export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, options: PlaywrightOptions = {}): Promise<{ serverProcess: ChildProcess, client: IDisposable, driver: IDriver }> {
|
||||
export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, options: PlaywrightOptions = {}, logger: Logger): Promise<{ serverProcess: ChildProcess, client: IDisposable, driver: IDriver }> {
|
||||
|
||||
// Launch server
|
||||
const { serverProcess, endpoint } = await launchServer(userDataDir, codeServerPath, extensionsPath, verbose);
|
||||
const { serverProcess, endpoint } = await launchServer(userDataDir, codeServerPath, extensionsPath, verbose, logger);
|
||||
|
||||
// Launch browser
|
||||
const { browser, context, page } = await launchBrowser(options, endpoint, workspacePath);
|
||||
const { browser, context, page } = await launchBrowser(options, endpoint, workspacePath, logger);
|
||||
|
||||
return {
|
||||
serverProcess,
|
||||
client: {
|
||||
dispose: () => { /* there is no client to dispose for browser, teardown is triggered via exitApplication call */ }
|
||||
},
|
||||
driver: new PlaywrightDriver(serverProcess, browser, context, page)
|
||||
driver: new PlaywrightDriver(serverProcess, browser, context, page, logger)
|
||||
};
|
||||
}
|
||||
|
||||
async function launchServer(userDataDir: string, codeServerPath: string | undefined, extensionsPath: string, verbose: boolean) {
|
||||
async function launchServer(userDataDir: string, codeServerPath: string | undefined, extensionsPath: string, verbose: boolean, logger: Logger) {
|
||||
const agentFolder = userDataDir;
|
||||
await promisify(mkdir)(agentFolder);
|
||||
const env = {
|
||||
|
|
@ -245,16 +238,16 @@ async function launchServer(userDataDir: string, codeServerPath: string | undefi
|
|||
args.push(`--logsPath=${logsPath}`);
|
||||
|
||||
if (verbose) {
|
||||
console.log(`Starting built server from '${serverLocation}'`);
|
||||
console.log(`Storing log files into '${logsPath}'`);
|
||||
logger.log(`Starting built server from '${serverLocation}'`);
|
||||
logger.log(`Storing log files into '${logsPath}'`);
|
||||
}
|
||||
} else {
|
||||
serverLocation = join(root, `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
||||
args.push('--logsPath', logsPath);
|
||||
|
||||
if (verbose) {
|
||||
console.log(`Starting server out of sources from '${serverLocation}'`);
|
||||
console.log(`Storing log files into '${logsPath}'`);
|
||||
logger.log(`Starting server out of sources from '${serverLocation}'`);
|
||||
logger.log(`Storing log files into '${logsPath}'`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -265,16 +258,16 @@ async function launchServer(userDataDir: string, codeServerPath: string | undefi
|
|||
);
|
||||
|
||||
if (verbose) {
|
||||
console.info(`*** Started server for browser smoke tests (pid: ${serverProcess.pid})`);
|
||||
serverProcess.once('exit', (code, signal) => console.info(`*** Server for browser smoke tests terminated (pid: ${serverProcess.pid}, code: ${code}, signal: ${signal})`));
|
||||
logger.log(`*** Started server for browser smoke tests (pid: ${serverProcess.pid})`);
|
||||
serverProcess.once('exit', (code, signal) => logger.log(`Server for browser smoke tests terminated (pid: ${serverProcess.pid}, code: ${code}, signal: ${signal})`));
|
||||
|
||||
serverProcess.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
|
||||
serverProcess.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
|
||||
serverProcess.stderr?.on('data', error => logger.log(`Server stderr: ${error}`));
|
||||
serverProcess.stdout?.on('data', data => logger.log(`Server stdout: ${data}`));
|
||||
}
|
||||
|
||||
process.on('exit', () => teardown(serverProcess));
|
||||
process.on('SIGINT', () => teardown(serverProcess));
|
||||
process.on('SIGTERM', () => teardown(serverProcess));
|
||||
process.on('exit', () => teardown(serverProcess, logger));
|
||||
process.on('SIGINT', () => teardown(serverProcess, logger));
|
||||
process.on('SIGTERM', () => teardown(serverProcess, logger));
|
||||
|
||||
return {
|
||||
serverProcess,
|
||||
|
|
@ -282,24 +275,24 @@ async function launchServer(userDataDir: string, codeServerPath: string | undefi
|
|||
};
|
||||
}
|
||||
|
||||
async function launchBrowser(options: PlaywrightOptions, endpoint: string, workspacePath: string) {
|
||||
async function launchBrowser(options: PlaywrightOptions, endpoint: string, workspacePath: string, logger: Logger) {
|
||||
const browser = await playwright[options.browser ?? 'chromium'].launch({ headless: options.headless ?? false });
|
||||
const context = await browser.newContext();
|
||||
|
||||
try {
|
||||
await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
|
||||
} catch (error) {
|
||||
console.warn(`Failed to start playwright tracing: ${error}`); // do not fail the build when this fails
|
||||
logger.log(`Failed to start playwright tracing: ${error}`); // do not fail the build when this fails
|
||||
}
|
||||
|
||||
const page = await context.newPage();
|
||||
await page.setViewportSize({ width, height });
|
||||
|
||||
page.on('pageerror', async (error) => console.error(`Playwright ERROR: page error: ${error}`));
|
||||
page.on('crash', page => console.error('Playwright ERROR: page crash'));
|
||||
page.on('pageerror', async (error) => logger.log(`Playwright ERROR: page error: ${error}`));
|
||||
page.on('crash', page => logger.log('Playwright ERROR: page crash'));
|
||||
page.on('response', async (response) => {
|
||||
if (response.status() >= 400) {
|
||||
console.error(`Playwright ERROR: HTTP status ${response.status()} for ${response.url()}`);
|
||||
logger.log(`Playwright ERROR: HTTP status ${response.status()} for ${response.url()}`);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -309,7 +302,7 @@ async function launchBrowser(options: PlaywrightOptions, endpoint: string, works
|
|||
return { browser, context, page };
|
||||
}
|
||||
|
||||
async function teardown(server: ChildProcess): Promise<void> {
|
||||
async function teardown(server: ChildProcess, logger: Logger): Promise<void> {
|
||||
const serverPid = server.pid;
|
||||
if (typeof serverPid !== 'number') {
|
||||
return;
|
||||
|
|
@ -324,14 +317,14 @@ async function teardown(server: ChildProcess): Promise<void> {
|
|||
} catch (error) {
|
||||
try {
|
||||
process.kill(serverPid, 0); // throws an exception if the process doesn't exist anymore
|
||||
console.warn(`Error tearing down server (pid: ${serverPid}, attempt: ${retries}): ${error}`);
|
||||
logger.log(`Error tearing down server (pid: ${serverPid}, attempt: ${retries}): ${error}`);
|
||||
} catch (error) {
|
||||
return; // Expected when process is gone
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.error(`Gave up tearing down server after ${retries} attempts...`);
|
||||
logger.log(`Gave up tearing down server after ${retries} attempts...`);
|
||||
}
|
||||
|
||||
function waitForEndpoint(server: ChildProcess): Promise<string> {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import { promisify } from 'util';
|
||||
import { gracefulify } from 'graceful-fs';
|
||||
import * as cp from 'child_process';
|
||||
import * as path from 'path';
|
||||
|
|
@ -12,7 +11,6 @@ import * as os from 'os';
|
|||
import * as minimist from 'minimist';
|
||||
import * as rimraf from 'rimraf';
|
||||
import * as mkdirp from 'mkdirp';
|
||||
import { ncp } from 'ncp';
|
||||
import * as vscodetest from 'vscode-test';
|
||||
import fetch from 'node-fetch';
|
||||
import { Quality, ApplicationOptions, MultiLogger, Logger, ConsoleLogger, FileLogger } from '../../automation';
|
||||
|
|
@ -59,7 +57,6 @@ const opts = minimist(args, {
|
|||
'wait-time',
|
||||
'test-repo',
|
||||
'screenshots',
|
||||
'log',
|
||||
'electronArgs'
|
||||
],
|
||||
boolean: [
|
||||
|
|
@ -209,9 +206,9 @@ else {
|
|||
|
||||
const userDataDir = path.join(testDataPath, 'd');
|
||||
|
||||
async function setupRepository(): Promise<void> {
|
||||
async function setupRepository(logger: Logger): Promise<void> {
|
||||
if (opts['test-repo']) {
|
||||
console.log('*** Copying test project repository:', opts['test-repo']);
|
||||
logger.log('Copying test project repository:', opts['test-repo']);
|
||||
rimraf.sync(workspacePath);
|
||||
// not platform friendly
|
||||
if (process.platform === 'win32') {
|
||||
|
|
@ -222,10 +219,10 @@ async function setupRepository(): Promise<void> {
|
|||
|
||||
} else {
|
||||
if (!fs.existsSync(workspacePath)) {
|
||||
console.log('*** Cloning test project repository...');
|
||||
logger.log('Cloning test project repository...');
|
||||
cp.spawnSync('git', ['clone', testRepoUrl, workspacePath]);
|
||||
} else {
|
||||
console.log('*** Cleaning test project repository...');
|
||||
logger.log('Cleaning test project repository...');
|
||||
cp.spawnSync('git', ['fetch'], { cwd: workspacePath });
|
||||
cp.spawnSync('git', ['reset', '--hard', 'FETCH_HEAD'], { cwd: workspacePath });
|
||||
cp.spawnSync('git', ['clean', '-xdf'], { cwd: workspacePath });
|
||||
|
|
@ -233,12 +230,12 @@ async function setupRepository(): Promise<void> {
|
|||
|
||||
// None of the current smoke tests have a dependency on the packages.
|
||||
// If new smoke tests are added that need the packages, uncomment this.
|
||||
// console.log('*** Running yarn...');
|
||||
// logger.log('Running yarn...');
|
||||
// cp.execSync('yarn', { cwd: workspacePath, stdio: 'inherit' });
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureStableCode(): Promise<void> {
|
||||
async function ensureStableCode(logger: Logger): Promise<void> {
|
||||
if (opts.web || !opts['build']) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -261,7 +258,7 @@ async function ensureStableCode(): Promise<void> {
|
|||
throw new Error(`Could not find suitable stable version ${majorMinorVersion}`);
|
||||
}
|
||||
|
||||
console.log(`*** Found VS Code v${version}, downloading previous VS Code version ${previousVersion.version}...`);
|
||||
logger.log(`Found VS Code v${version}, downloading previous VS Code version ${previousVersion.version}...`);
|
||||
|
||||
const stableCodeExecutable = await vscodetest.download({
|
||||
cachePath: path.join(os.tmpdir(), 'vscode-test'),
|
||||
|
|
@ -281,40 +278,22 @@ async function ensureStableCode(): Promise<void> {
|
|||
throw new Error(`Can't find Stable VSCode at ${stableCodePath}.`);
|
||||
}
|
||||
|
||||
console.log(`*** Using stable build ${stableCodePath} for migration tests`);
|
||||
logger.log(`Using stable build ${stableCodePath} for migration tests`);
|
||||
|
||||
opts['stable-build'] = stableCodePath;
|
||||
}
|
||||
|
||||
async function setup(): Promise<void> {
|
||||
console.log('*** Test data:', testDataPath);
|
||||
console.log('*** Preparing smoketest setup...');
|
||||
async function setup(logger: Logger): Promise<void> {
|
||||
logger.log('Test data:', testDataPath);
|
||||
logger.log('Preparing smoketest setup...');
|
||||
|
||||
await ensureStableCode();
|
||||
await setupRepository();
|
||||
await ensureStableCode(logger);
|
||||
await setupRepository(logger);
|
||||
|
||||
console.log('*** Smoketest setup done!\n');
|
||||
logger.log('Smoketest setup done!\n');
|
||||
}
|
||||
|
||||
async function createOptions(): Promise<ApplicationOptions> {
|
||||
const loggers: Logger[] = [];
|
||||
|
||||
if (opts.verbose) {
|
||||
loggers.push(new ConsoleLogger());
|
||||
}
|
||||
|
||||
const log: string | undefined = 'trace'; // because smoke tests are flaky
|
||||
let logsPath: string;
|
||||
if (opts.log) {
|
||||
logsPath = opts.log;
|
||||
} else {
|
||||
logsPath = path.join(repoPath, '.build', 'logs', opts.web ? 'smoke-tests-browser' : opts.remote ? 'smoke-tests-remote' : 'smoke-tests', 'smoke-test-runner.log');
|
||||
}
|
||||
|
||||
await mkdirp(path.dirname(logsPath));
|
||||
|
||||
loggers.push(new FileLogger(logsPath));
|
||||
|
||||
return {
|
||||
quality,
|
||||
codePath: opts.build,
|
||||
|
|
@ -322,9 +301,8 @@ async function createOptions(): Promise<ApplicationOptions> {
|
|||
userDataDir,
|
||||
extensionsPath,
|
||||
waitTime: parseInt(opts['wait-time'] || '0') || 20,
|
||||
logger: new MultiLogger(loggers),
|
||||
logger: await createLogger(),
|
||||
verbose: opts.verbose,
|
||||
log,
|
||||
screenshotsPath,
|
||||
remote: opts.remote,
|
||||
web: opts.web,
|
||||
|
|
@ -334,19 +312,31 @@ async function createOptions(): Promise<ApplicationOptions> {
|
|||
};
|
||||
}
|
||||
|
||||
|
||||
async function createLogger(): Promise<Logger> {
|
||||
const loggers: Logger[] = [];
|
||||
|
||||
// Log to console if verbose
|
||||
if (opts.verbose) {
|
||||
loggers.push(new ConsoleLogger());
|
||||
}
|
||||
|
||||
// Always log to log file
|
||||
const logPath = path.join(repoPath, '.build', 'logs', opts.web ? 'smoke-tests-browser' : opts.remote ? 'smoke-tests-remote' : 'smoke-tests');
|
||||
await mkdirp(logPath);
|
||||
loggers.push(new FileLogger(path.join(logPath, 'smoke-test-runner.log')));
|
||||
|
||||
return new MultiLogger(loggers);
|
||||
}
|
||||
|
||||
before(async function () {
|
||||
this.timeout(2 * 60 * 1000); // allow two minutes for setup
|
||||
await setup();
|
||||
this.defaultOptions = await createOptions();
|
||||
const options = this.defaultOptions = await createOptions();
|
||||
|
||||
await setup(options.logger);
|
||||
});
|
||||
|
||||
after(async function () {
|
||||
if (opts.log) {
|
||||
const logsDir = path.join(userDataDir, 'logs');
|
||||
const destLogsDir = path.join(path.dirname(opts.log), 'logs');
|
||||
await promisify(ncp)(logsDir, destLogsDir);
|
||||
}
|
||||
|
||||
try {
|
||||
// TODO@tyriar TODO@meganrogge lately deleting the test root
|
||||
// folder results in timeouts of 60s or EPERM issues which
|
||||
|
|
@ -376,7 +366,7 @@ after(async function () {
|
|||
})
|
||||
]);
|
||||
} catch (error) {
|
||||
console.error(`Unable to delete smoke test workspace: ${error}. This indicates some process is locking the workspace folder.`);
|
||||
this.options.logger(`Unable to delete smoke test workspace: ${error}. This indicates some process is locking the workspace folder.`);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -58,8 +58,8 @@ export async function startApp(args: minimist.ParsedArgs, options: ApplicationOp
|
|||
|
||||
await app.start();
|
||||
|
||||
if (args.log && options.testTitle) {
|
||||
app.logger.log('*** Test start:', options.testTitle);
|
||||
if (options.testTitle) {
|
||||
app.logger.log('Test start:', options.testTitle);
|
||||
}
|
||||
|
||||
return app;
|
||||
|
|
|
|||
|
|
@ -35,4 +35,37 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
|
|||
|
||||
const mocha = new Mocha(options);
|
||||
mocha.addFile('out/main.js');
|
||||
mocha.run(failures => process.exit(failures ? -1 : 0));
|
||||
mocha.run(failures => {
|
||||
|
||||
// Indicate location of log files for further diagnosis
|
||||
if (failures) {
|
||||
const repoPath = path.join(__dirname, '..', '..', '..');
|
||||
const logPath = path.join(repoPath, '.build', 'logs', opts.web ? 'smoke-tests-browser' : opts.remote ? 'smoke-tests-remote' : 'smoke-tests');
|
||||
const logFile = path.join(logPath, 'smoke-test-runner.log');
|
||||
|
||||
if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
|
||||
console.log(`
|
||||
###################################################################
|
||||
# #
|
||||
# Logs are attached as build artefact and can be downloaded #
|
||||
# from the build Summary page (Summary -> Related -> N published) #
|
||||
# #
|
||||
###################################################################
|
||||
`);
|
||||
} else {
|
||||
console.log(`
|
||||
#############################################
|
||||
#
|
||||
# Log files of client & server are stored into
|
||||
# '${logPath}'.
|
||||
#
|
||||
# Logs of the smoke test runner are stored into
|
||||
# '${logFile}'.
|
||||
#
|
||||
#############################################
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(failures ? -1 : 0);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue