mirror of
https://github.com/Microsoft/vscode
synced 2024-08-27 13:00:58 +00:00
Merge branch 'smoketest'
This commit is contained in:
commit
750b49ce9e
|
@ -58,4 +58,5 @@ function npmInstallBuildDependencies() {
|
|||
}
|
||||
|
||||
npmInstall(`build`); // node modules required for build
|
||||
npmInstall('test/smoke'); // node modules required for smoketest
|
||||
npmInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron
|
|
@ -24,5 +24,5 @@ step "Build minified & upload source maps" \
|
|||
step "Run smoke test" \
|
||||
pushd test/smoke
|
||||
npm install
|
||||
npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
|
||||
npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
|
||||
popd
|
|
@ -36,7 +36,7 @@ function configureEnvironment {
|
|||
function runTest {
|
||||
pushd test/smoke
|
||||
npm install
|
||||
sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
|
||||
sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
|
||||
popd
|
||||
}
|
||||
|
||||
|
|
|
@ -40,7 +40,7 @@ step "Build minified" {
|
|||
step "Run smoke test" {
|
||||
exec { & Push-Location test\smoke }
|
||||
exec { & npm install }
|
||||
exec { & npm test -- --latest "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
|
||||
exec { & npm run smoketest -- "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
|
||||
exec { & Pop-Location }
|
||||
}
|
||||
|
||||
|
|
|
@ -19,7 +19,8 @@
|
|||
"precommit": "node build/gulpfile.hygiene.js",
|
||||
"gulp": "gulp --max_old_space_size=4096",
|
||||
"7z": "7z",
|
||||
"update-grammars": "node build/npm/update-all-grammars.js"
|
||||
"update-grammars": "node build/npm/update-all-grammars.js",
|
||||
"smoketest": "node test/smoke/out/main.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"applicationinsights": "0.17.1",
|
||||
|
|
|
@ -1,9 +1,14 @@
|
|||
# VS Code Smoke Testing
|
||||
|
||||
- Run `npm install`
|
||||
- Start the tests: `npm test -- --latest "path/to/binary"`.
|
||||
```
|
||||
npm rum smoketest -- "path/to/code"
|
||||
```
|
||||
|
||||
If you want to include 'Data Migration' area tests use `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively.
|
||||
If you want to include 'Data Migration' area tests use:
|
||||
|
||||
```
|
||||
npm run smoketest -- "path/to/code" "path/to/codeStable"
|
||||
```
|
||||
|
||||
Detailed prerequisites and running steps are described [in our smoke test wiki](https://github.com/Microsoft/vscode/wiki/Smoke-Test#automated-smoke-test).
|
||||
|
||||
|
@ -50,15 +55,7 @@ To add new test, `./test/${area}.ts` should be updated. The same instruction-sty
|
|||
Almost on every automated test action it captures a screenshot. These help to determine an issue, if smoke test fails. The normal workflow is that you understand what code is doing and then try to match it up with screenshots obtained from the test.
|
||||
|
||||
# Running "Out of Sources"
|
||||
If you did a fix in VS Code that you need in order for the smoke test to succeed, here is how you can run the smoke test against the sources of VS Code:
|
||||
* Set related environment variables in the console:
|
||||
* `export NODE_ENV=development`
|
||||
* `export VSCODE_DEV=1`
|
||||
* `export VSCODE_CLI=1`
|
||||
* open `application.ts`
|
||||
* pass in the vscode folder as argument to the application
|
||||
* e.g. instead of `args: args` type `args: ['/Users/bpasero/Development/vscode', ...args]`
|
||||
* `cd test/smoke`
|
||||
* `npm install`
|
||||
* `npm test -- --latest <path to electron>`
|
||||
* e.g. on macOS: `npm test -- --latest <path to vscode>/.build/electron/Code\ -\ OSS.app/Contents/MacOS/Electron`
|
||||
|
||||
```
|
||||
npm run smoketest
|
||||
```
|
|
@ -3,24 +3,22 @@
|
|||
"version": "0.1.0",
|
||||
"main": "./src/main.js",
|
||||
"scripts": {
|
||||
"compile": "tsc",
|
||||
"pretest": "tsc",
|
||||
"test": "node out/main.js"
|
||||
"postinstall": "tsc",
|
||||
"watch": "tsc --watch"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/mocha": "^2.2.41",
|
||||
"@types/node": "^6.0.70",
|
||||
"@types/webdriverio": "^4.6.1",
|
||||
"@types/electron": "~1.4.37",
|
||||
"@types/rimraf": "^0.0.28",
|
||||
"@types/htmlparser2": "^3.7.29",
|
||||
"@types/mkdirp": "^0.5.1",
|
||||
"@types/mocha": "^2.2.41",
|
||||
"@types/node": "^8.0.26",
|
||||
"@types/rimraf": "^0.0.28",
|
||||
"@types/webdriverio": "^4.6.1",
|
||||
"htmlparser2": "^3.9.2",
|
||||
"mocha": "^3.2.0",
|
||||
"spectron": "~3.6.4",
|
||||
"typescript": "^2.2.2",
|
||||
"rimraf": "^2.6.1",
|
||||
"commander": "^2.9.0",
|
||||
"simple-git": "^1.73.0",
|
||||
"spectron": "~3.6.4",
|
||||
"strip-json-comments": "^2.0.1",
|
||||
"htmlparser2": "^3.9.2"
|
||||
"typescript": "^2.2.2"
|
||||
}
|
||||
}
|
|
@ -3,220 +3,132 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
const fs = require('fs');
|
||||
const https = require('https');
|
||||
const program = require('commander');
|
||||
const git = require('simple-git')();
|
||||
const child_process = require('child_process');
|
||||
const path = require('path');
|
||||
const mkdirp = require('mkdirp');
|
||||
import * as fs from 'fs';
|
||||
import * as https from 'https';
|
||||
import * as cp from 'child_process';
|
||||
import * as path from 'path';
|
||||
import * as mkdirp from 'mkdirp';
|
||||
|
||||
const testDataPath = path.join(process.cwd(), 'test_data');
|
||||
const codeWorkspacePath = path.join(testDataPath, 'smoketest.code-workspace');
|
||||
const testDataPath = path.join(__dirname, '..', 'test_data');
|
||||
const workspacePath = path.join(testDataPath, 'smoketest.code-workspace');
|
||||
const testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express';
|
||||
const testRepoLocalDir = path.join(testDataPath, 'vscode-smoketest-express');
|
||||
const keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings';
|
||||
|
||||
mkdirp.sync(testDataPath);
|
||||
|
||||
program
|
||||
.option('-l, --latest <file path>', 'path to the latest VS Code to test')
|
||||
.option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests');
|
||||
|
||||
program.on('--help', () => {
|
||||
console.log(' Examples:');
|
||||
console.log('');
|
||||
console.log(' $ npm test -- --latest path/to/binary');
|
||||
console.log(' $ npm test -- -l path/to/binary');
|
||||
console.log('');
|
||||
console.log(' $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary');
|
||||
console.log(' $ npm test -- -l path/to/latest/binary -s path/to/stable/binary');
|
||||
console.log('');
|
||||
});
|
||||
program.parse(process.argv);
|
||||
|
||||
if (!program.latest) {
|
||||
fail('You must specify the binary to run the smoke test against');
|
||||
}
|
||||
if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) {
|
||||
fail('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.');
|
||||
}
|
||||
if (parseInt(process.version.substr(1)) < 6) {
|
||||
fail('Please update your Node version to greater than 6 to run the smoke test.');
|
||||
}
|
||||
|
||||
// Setting up environment variables
|
||||
process.env.VSCODE_LATEST_PATH = program.latest;
|
||||
if (program.stable) {
|
||||
process.env.VSCODE_STABLE_PATH = program.stable;
|
||||
}
|
||||
process.env.SMOKETEST_REPO = testRepoLocalDir;
|
||||
if (program.latest && (program.latest.indexOf('Code - Insiders') /* macOS/Windows */ || program.latest.indexOf('code-insiders') /* Linux */) >= 0) {
|
||||
process.env.VSCODE_EDITION = 'insiders';
|
||||
}
|
||||
process.env.VSCODE_WORKSPACE_PATH = codeWorkspacePath;
|
||||
|
||||
// Setting up 'vscode-smoketest-express' project
|
||||
let os = process.platform.toString();
|
||||
if (os === 'darwin') {
|
||||
os = 'osx';
|
||||
}
|
||||
else if (os === 'win32') {
|
||||
os = 'win';
|
||||
}
|
||||
|
||||
main().catch(err => console.error(err));
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, path.join(testDataPath, 'keybindings.json'));
|
||||
|
||||
const workspace = {
|
||||
id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
|
||||
folders: [
|
||||
toUri(path.join(testRepoLocalDir, 'public')),
|
||||
toUri(path.join(testRepoLocalDir, 'routes')),
|
||||
toUri(path.join(testRepoLocalDir, 'views'))
|
||||
]
|
||||
};
|
||||
|
||||
await createWorkspaceFile(codeWorkspacePath, workspace);
|
||||
await cleanOrClone(testRepoUrl, testRepoLocalDir);
|
||||
await execute('npm install', testRepoLocalDir);
|
||||
await runTests();
|
||||
}
|
||||
|
||||
function fail(errorMessage): void {
|
||||
console.error(errorMessage);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (parseInt(process.version.substr(1)) < 6) {
|
||||
fail('Please update your Node version to greater than 6 to run the smoke test.');
|
||||
}
|
||||
|
||||
const repoPath = path.join(__dirname, '..', '..', '..');
|
||||
|
||||
function getDevElectronPath(): string {
|
||||
const buildPath = path.join(repoPath, '.build');
|
||||
const product = require(path.join(repoPath, 'product.json'));
|
||||
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return path.join(buildPath, `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
|
||||
case 'linux':
|
||||
return path.join(buildPath, 'electron', `${product.applicationName}`);
|
||||
case 'win32':
|
||||
return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
|
||||
default:
|
||||
throw new Error('Unsupported platform.');
|
||||
}
|
||||
}
|
||||
|
||||
let [, , testCodePath, stableCodePath] = process.argv;
|
||||
|
||||
if (testCodePath) {
|
||||
process.env.VSCODE_PATH = testCodePath;
|
||||
|
||||
if (stableCodePath) {
|
||||
process.env.VSCODE_STABLE_PATH = stableCodePath;
|
||||
}
|
||||
} else {
|
||||
testCodePath = getDevElectronPath();
|
||||
process.env.VSCODE_PATH = testCodePath;
|
||||
process.env.VSCODE_REPOSITORY = repoPath;
|
||||
process.env.VSCODE_DEV = '1';
|
||||
process.env.VSCODE_CLI = '1';
|
||||
}
|
||||
|
||||
if (!fs.existsSync(testCodePath)) {
|
||||
fail(`Can't find Code at ${testCodePath}.`);
|
||||
}
|
||||
|
||||
process.env.SMOKETEST_REPO = testRepoLocalDir;
|
||||
process.env.VSCODE_WORKSPACE_PATH = workspacePath;
|
||||
|
||||
if ((testCodePath.indexOf('Code - Insiders') /* macOS/Windows */ || testCodePath.indexOf('code-insiders') /* Linux */) >= 0) {
|
||||
process.env.VSCODE_EDITION = 'insiders';
|
||||
}
|
||||
|
||||
function getKeybindingPlatform(): string {
|
||||
switch (process.platform) {
|
||||
case 'darwin': return 'osx';
|
||||
case 'win32': return 'win';
|
||||
default: return process.platform;
|
||||
}
|
||||
}
|
||||
|
||||
function toUri(path: string): string {
|
||||
if (os === 'win') {
|
||||
if (process.platform === 'win32') {
|
||||
return `file:///${path.replace(/\\/g, '/')}`;
|
||||
}
|
||||
|
||||
return `file://${path}`;
|
||||
}
|
||||
|
||||
function runTests(): void {
|
||||
console.log('Running tests...');
|
||||
var proc = child_process.spawn(process.execPath, [
|
||||
'out/mocha-runner.js'
|
||||
]);
|
||||
proc.stdout.on('data', data => {
|
||||
console.log(data.toString());
|
||||
});
|
||||
proc.stderr.on('data', data => {
|
||||
var date = new Date().toLocaleString();
|
||||
fs.appendFile(path.join(testDataPath, 'errors.log'), `${date}: ${data.toString()}`, (err) => {
|
||||
if (err) {
|
||||
throw new Error(`Could not write stderr to errors.log with the following error: ${err}`);
|
||||
};
|
||||
});
|
||||
});
|
||||
proc.on('exit', (code) => {
|
||||
process.exit(code);
|
||||
});
|
||||
}
|
||||
async function main(): Promise<void> {
|
||||
const keybindingsUrl = `https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.${getKeybindingPlatform()}.json`;
|
||||
console.log(`Fetching keybindings from ${keybindingsUrl}...`);
|
||||
|
||||
async function cleanOrClone(repo: string, dir: string): Promise<any> {
|
||||
console.log('Cleaning or cloning test project repository...');
|
||||
await new Promise((c, e) => {
|
||||
https.get(keybindingsUrl, res => {
|
||||
const output = fs.createWriteStream(path.join(testDataPath, 'keybindings.json'));
|
||||
res.on('error', e);
|
||||
output.on('error', e);
|
||||
output.on('close', c);
|
||||
res.pipe(output);
|
||||
}).on('error', e);
|
||||
});
|
||||
|
||||
if (!folderExists(dir)) {
|
||||
await gitClone(repo, dir);
|
||||
if (!fs.existsSync(workspacePath)) {
|
||||
console.log('Creating workspace file...');
|
||||
const workspace = {
|
||||
id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
|
||||
folders: [
|
||||
toUri(path.join(testRepoLocalDir, 'public')),
|
||||
toUri(path.join(testRepoLocalDir, 'routes')),
|
||||
toUri(path.join(testRepoLocalDir, 'views'))
|
||||
]
|
||||
};
|
||||
|
||||
fs.writeFileSync(workspacePath, JSON.stringify(workspace, null, '\t'));
|
||||
}
|
||||
|
||||
if (!fs.existsSync(testRepoLocalDir)) {
|
||||
console.log('Cloning test project repository...');
|
||||
cp.spawnSync('git', ['clone', testRepoUrl, testRepoLocalDir]);
|
||||
} else {
|
||||
git.cwd(dir);
|
||||
await new Promise((c, e) => git.fetch(err => err ? e(err) : c()));
|
||||
await gitResetAndClean();
|
||||
console.log('Cleaning test project repository...');
|
||||
cp.spawnSync('git', ['fetch'], { cwd: testRepoLocalDir });
|
||||
cp.spawnSync('git', ['reset', '--hard', 'FETCH_HEAD'], { cwd: testRepoLocalDir });
|
||||
cp.spawnSync('git', ['clean', '-xdf'], { cwd: testRepoLocalDir });
|
||||
}
|
||||
|
||||
console.log('Running npm install...');
|
||||
cp.execSync('npm install', { cwd: testRepoLocalDir, stdio: 'inherit' });
|
||||
|
||||
console.log('Running tests...');
|
||||
const mocha = cp.spawnSync(process.execPath, ['out/mocha-runner.js'], { cwd: path.join(__dirname, '..'), stdio: 'inherit' });
|
||||
process.exit(mocha.status);
|
||||
}
|
||||
|
||||
function gitClone(repo: string, dir: string): Promise<any> {
|
||||
return new Promise((res, rej) => {
|
||||
git.clone(repo, dir, () => {
|
||||
console.log('Test repository successfully cloned.');
|
||||
res();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function gitResetAndClean(): Promise<any> {
|
||||
await new Promise((c, e) => git.reset(['FETCH_HEAD', '--hard'], err => err ? e(err) : c()));
|
||||
await new Promise((c, e) => git.clean('f', ['-d'], err => err ? e(err) : c()));
|
||||
console.log('Test project was successfully reset to initial state.');
|
||||
}
|
||||
|
||||
function execute(cmd: string, dir: string): Promise<any> {
|
||||
return new Promise((res, rej) => {
|
||||
console.log(`Running ${cmd}...`);
|
||||
child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
rej(error);
|
||||
}
|
||||
if (stderr) {
|
||||
console.error(stderr);
|
||||
}
|
||||
console.log(stdout);
|
||||
res();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getKeybindings(url: string, location: string): Promise<any> {
|
||||
console.log(`Fetching keybindings from ${url}...`);
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, (res) => {
|
||||
if (res.statusCode !== 200) {
|
||||
reject(`Failed to obtain key bindings with response code: ${res.statusCode}`);
|
||||
}
|
||||
|
||||
var buffer: Buffer[] = [];
|
||||
res.on('data', (chunk) => buffer.push(chunk));
|
||||
res.on('end', () => {
|
||||
fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => {
|
||||
console.log('Keybindings were successfully fetched.');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}).on('error', (e) => {
|
||||
reject(`Failed to obtain key bindings with an error: ${e}`);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function createWorkspaceFile(path: string, workspace: any): Promise<any> {
|
||||
console.log(`Creating workspace file at ${path}...`);
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.exists(path, exists => {
|
||||
if (exists) {
|
||||
return resolve();
|
||||
}
|
||||
|
||||
fs.writeFile(path, JSON.stringify(workspace, null, '\t'), error => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function folderExists(folder: string): boolean {
|
||||
try {
|
||||
fs.accessSync(folder, 'rw');
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function binaryExists(filePath: string): boolean {
|
||||
try {
|
||||
fs.accessSync(filePath, 'x');
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
main().catch(fail);
|
|
@ -4,13 +4,14 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
const MochaTest = require('mocha');
|
||||
const path = require('path');
|
||||
|
||||
const mochaTest = new MochaTest({
|
||||
timeout: 60000,
|
||||
slow: 10000,
|
||||
useColors: true
|
||||
});
|
||||
mochaTest.addFile(require('path').join(process.cwd(), 'out/test.js'));
|
||||
mochaTest.addFile(path.join(__dirname, 'test.js'));
|
||||
mochaTest.run((failures) => {
|
||||
process.exit(failures);
|
||||
});
|
|
@ -9,10 +9,10 @@ import { Screenshot } from '../helpers/screenshot';
|
|||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
export const LATEST_PATH = process.env.VSCODE_LATEST_PATH;
|
||||
export const STABLE_PATH = process.env.VSCODE_STABLE_PATH;
|
||||
export const WORKSPACE_PATH = process.env.SMOKETEST_REPO;
|
||||
export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH;
|
||||
export const LATEST_PATH = process.env.VSCODE_PATH || '';
|
||||
export const STABLE_PATH = process.env.VSCODE_STABLE_PATH || '';
|
||||
export const WORKSPACE_PATH = process.env.SMOKETEST_REPO || '';
|
||||
export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH || '';
|
||||
export const USER_DIR = 'test_data/temp_user_dir';
|
||||
export const EXTENSIONS_DIR = 'test_data/temp_extensions_dir';
|
||||
|
||||
|
@ -50,6 +50,11 @@ export class SpectronApplication {
|
|||
args.push(`--extensions-dir=${this.sampleExtensionsDir}`);
|
||||
}
|
||||
|
||||
const repo = process.env.VSCODE_REPOSITORY;
|
||||
if (repo) {
|
||||
args = [repo, ...args];
|
||||
}
|
||||
|
||||
this.spectron = new Application({
|
||||
path: electronPath,
|
||||
args: args,
|
||||
|
@ -96,7 +101,7 @@ export class SpectronApplication {
|
|||
}
|
||||
|
||||
private retrieveKeybindings() {
|
||||
fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => {
|
||||
fs.readFile(path.join(__dirname, '../../test_data/keybindings.json'), 'utf8', (err, data) => {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
|
|
1
test_data/vscode-smoketest-express
Submodule
1
test_data/vscode-smoketest-express
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit 5ac7ff24e8c4028e00d8b7d093a920db31513d2e
|
Loading…
Reference in a new issue