development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-11-05 18:29:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Produce logs during test runs (integration, smoke) and publish as artefacts (#126065)

Browse source 
* first cut upload log files from integration test runs

* always publish log files

* move logs into integration-tests folder

* also produce logs for smoke tests

* store remote log files

* wire in logspath arg properly

* always publish logs

* enable logs for browser based smoke tests

* log only in verbose
This commit is contained in:
Benjamin Pasero 2021-06-15 08:33:26 +02:00 committed by Johannes Rieken
parent 0e35106779
commit acc2a805ec
No known key found for this signature in database
GPG key ID: 96634B5AF12F8798
8 changed files with 68 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
build/azure-pipelines/darwin/product-build-darwin.yml
View file

@ -253,6 +253,14 @@ steps:
continueOnError: true
condition: failed()
- task: PublishPipelineArtifact@0
inputs:
artifactName: logs-macos-$(VSCODE_ARCH)
targetPath: .build/logs
displayName: "Publish Log Files"
continueOnError: true
condition: succeededOrFailed()
- task: PublishTestResults@2
displayName: Publish Tests Results
inputs:

10
build/azure-pipelines/linux/product-build-linux.yml
View file

@ -205,12 +205,20 @@ steps:
- task: PublishPipelineArtifact@0
inputs:
artifactName: "crash-dump-linux-$(VSCODE_ARCH)"
artifactName: crash-dump-linux-$(VSCODE_ARCH)
targetPath: .build/crashes
displayName: "Publish Crash Reports"
continueOnError: true
condition: failed()
- task: PublishPipelineArtifact@0
inputs:
artifactName: logs-linux-$(VSCODE_ARCH)
targetPath: .build/logs
displayName: "Publish Log Files"
continueOnError: true
condition: succeededOrFailed()
- task: PublishTestResults@2
displayName: Publish Tests Results
inputs:

8
build/azure-pipelines/win32/product-build-win32.yml
View file

@ -195,6 +195,14 @@ steps:
continueOnError: true
condition: failed()
- task: PublishPipelineArtifact@0
inputs:
artifactName: logs-windows-$(VSCODE_ARCH)
targetPath: .build\logs
displayName: "Publish Log Files"
continueOnError: true
condition: succeededOrFailed()
- task: PublishTestResults@2
displayName: Publish Tests Results
inputs:

5
scripts/test-integration.bat
View file

@ -5,6 +5,7 @@ pushd %~dp0\..
set VSCODEUSERDATADIR=%TEMP%\vscodeuserfolder-%RANDOM%-%TIME:~6,2%
set VSCODECRASHDIR=%~dp0\..\.build\crashes
set VSCODELOGSDIR=%~dp0\..\.build\logs\integration-tests
:: Figure out which Electron to use for running tests
if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
@ -14,6 +15,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
set VSCODE_BUILD_BUILTIN_EXTENSIONS_SILENCE_PLEASE=1
echo Storing crash reports into '%VSCODECRASHDIR%'.
echo Storing log files into '%VSCODELOGSDIR%'.
echo Running integration tests out of sources.
) else (
:: Run from a built: need to compile all test extensions
@ -37,6 +39,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
set ELECTRON_ENABLE_LOGGING=1
echo Storing crash reports into '%VSCODECRASHDIR%'.
echo Storing log files into '%VSCODELOGSDIR%'.
echo Running integration tests with '%INTEGRATION_TEST_ELECTRON_PATH%' as build.
)
@ -46,7 +49,7 @@ if %errorlevel% neq 0 exit /b %errorlevel%
:: Tests in the extension host
set ALL_PLATFORMS_API_TESTS_EXTRA_ARGS=--disable-telemetry --skip-welcome --crash-reporter-directory=%VSCODECRASHDIR% --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=%VSCODEUSERDATADIR%
set ALL_PLATFORMS_API_TESTS_EXTRA_ARGS=--disable-telemetry --skip-welcome --crash-reporter-directory=%VSCODECRASHDIR% --logsPath=%VSCODELOGSDIR% --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=%VSCODEUSERDATADIR%
call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\vscode-api-tests\testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%~dp0\..\extensions\vscode-api-tests --extensionTestsPath=%~dp0\..\extensions\vscode-api-tests\out\singlefolder-tests %ALL_PLATFORMS_API_TESTS_EXTRA_ARGS%
if %errorlevel% neq 0 exit /b %errorlevel%

5
scripts/test-integration.sh
View file

@ -14,6 +14,7 @@ fi
VSCODEUSERDATADIR=`mktemp -d 2>/dev/null`
VSCODECRASHDIR=$ROOT/.build/crashes
VSCODELOGSDIR=$ROOT/.build/logs/integration-tests
cd $ROOT
# Figure out which Electron to use for running tests
@ -23,6 +24,7 @@ then
INTEGRATION_TEST_ELECTRON_PATH="./scripts/code.sh"
echo "Storing crash reports into '$VSCODECRASHDIR'."
echo "Storing log files into '$VSCODELOGSDIR'."
echo "Running integration tests out of sources."
else
# Run from a built: need to compile all test extensions
@ -47,6 +49,7 @@ else
export ELECTRON_ENABLE_LOGGING=1
echo "Storing crash reports into '$VSCODECRASHDIR'."
echo "Storing log files into '$VSCODELOGSDIR'."
echo "Running integration tests with '$INTEGRATION_TEST_ELECTRON_PATH' as build."
fi
@ -62,7 +65,7 @@ after_suite
# Tests in the extension host
ALL_PLATFORMS_API_TESTS_EXTRA_ARGS="--disable-telemetry --skip-welcome --crash-reporter-directory=$VSCODECRASHDIR --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=$VSCODEUSERDATADIR"
ALL_PLATFORMS_API_TESTS_EXTRA_ARGS="--disable-telemetry --skip-welcome --crash-reporter-directory=$VSCODECRASHDIR --logsPath=$VSCODELOGSDIR --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=$VSCODEUSERDATADIR"
"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_EXTRA_ARGS $ROOT/extensions/vscode-api-tests/testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=$ROOT/extensions/vscode-api-tests --extensionTestsPath=$ROOT/extensions/vscode-api-tests/out/singlefolder-tests $ALL_PLATFORMS_API_TESTS_EXTRA_ARGS
after_suite

3
test/automation/src/code.ts
View file

@ -123,7 +123,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
copyExtension(options.extensionsPath, 'vscode-notebook-tests');
if (options.web) {
await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath);
await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath, Boolean(options.verbose));
connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, options.browser);
return connect(connectDriver, child, '', handle, options.logger);
}
@ -144,6 +144,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
'--disable-workspace-trust',
`--extensions-dir=${options.extensionsPath}`,
`--user-data-dir=${options.userDataDir}`,
`--logsPath=${path.join(repoPath, '.build', 'logs', 'smoke-tests')}`,
'--driver', handle
];

25
test/automation/src/playwrightDriver.ts
View file

@ -94,7 +94,7 @@ let server: ChildProcess | undefined;
let endpoint: string | undefined;
let workspacePath: string | undefined;
export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, extPath: string): Promise<void> {
export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, extPath: string, verbose: boolean): Promise<void> {
workspacePath = _workspacePath;
const agentFolder = userDataDir;
@ -104,25 +104,42 @@ export async function launch(userDataDir: string, _workspacePath: string, codeSe
VSCODE_REMOTE_SERVER_PATH: codeServerPath,
...process.env
};
const root = join(__dirname, '..', '..', '..');
const logsPath = join(root, '.build', 'logs', 'smoke-tests-browser');
const args = ['--browser', 'none', '--driver', 'web', '--extensions-dir', extPath];
let serverLocation: string | undefined;
if (codeServerPath) {
serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
args.push(`--logsPath=${logsPath}`);
console.log(`Starting built server from '${serverLocation}'`);
console.log(`Storing log files into '${logsPath}'`);
} else {
serverLocation = join(__dirname, '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
serverLocation = join(root, `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
args.push('--logsPath', logsPath);
console.log(`Starting server out of sources from '${serverLocation}'`);
console.log(`Storing log files into '${logsPath}'`);
}
server = spawn(
serverLocation,
args,
{ env }
);
server.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
server.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
if (verbose) {
server.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
server.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
}
process.on('exit', teardown);
process.on('SIGINT', teardown);
process.on('SIGTERM', teardown);
endpoint = await waitForEndpoint();
}

16
test/integration/browser/src/index.ts
View file

@ -92,27 +92,35 @@ async function launchServer(browserType: BrowserType): Promise<{ endpoint: url.U
...process.env
};
const root = path.join(__dirname, '..', '..', '..', '..');
const logsPath = path.join(root, '.build', 'logs', 'integration-tests-browser');
const serverArgs = ['--browser', 'none', '--driver', 'web', '--enable-proposed-api'];
let serverLocation: string;
if (process.env.VSCODE_REMOTE_SERVER_PATH) {
serverLocation = path.join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
serverArgs.push(`--logsPath=${logsPath}`);
console.log(`Starting built server from '${serverLocation}'`);
console.log(`Storing log files into '${logsPath}'`);
} else {
serverLocation = path.join(__dirname, '..', '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
serverLocation = path.join(root, `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
serverArgs.push('--logsPath', logsPath);
process.env.VSCODE_DEV = '1';
console.log(`Starting server out of sources from '${serverLocation}'`);
console.log(`Storing log files into '${logsPath}'`);
}
let serverProcess = cp.spawn(
serverLocation,
['--browser', 'none', '--driver', 'web', '--enable-proposed-api'],
serverArgs,
{ env }
);
serverProcess?.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
if (optimist.argv.debug) {
serverProcess?.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
serverProcess?.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
}