development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-27 13:00:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'smoketest'

Browse source
This commit is contained in:
Joao 2017-09-04 11:32:58 +02:00
parent cb5d03296f 17bf788e41
commit 750b49ce9e
11 changed files with 147 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
build/npm/postinstall.js
View file

@ -58,4 +58,5 @@ function npmInstallBuildDependencies() {
}
npmInstall(`build`); // node modules required for build
npmInstall('test/smoke'); // node modules required for smoketest
npmInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron

2
build/tfs/darwin/smoketest.sh
View file

@ -24,5 +24,5 @@ step "Build minified & upload source maps" \
step "Run smoke test" \
pushd test/smoke
npm install
npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
popd

2
build/tfs/linux/smoketest.sh
View file

@ -36,7 +36,7 @@ function configureEnvironment {
function runTest {
pushd test/smoke
npm install
sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
popd
}

2
build/tfs/win32/smoketest.ps1
View file

@ -40,7 +40,7 @@ step "Build minified" {
step "Run smoke test" {
exec { & Push-Location test\smoke }
exec { & npm install }
exec { & npm test -- --latest "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
exec { & npm run smoketest -- "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
exec { & Pop-Location }
}

3
package.json
View file

@ -19,7 +19,8 @@
"precommit": "node build/gulpfile.hygiene.js",
"gulp": "gulp --max_old_space_size=4096",
"7z": "7z",
"update-grammars": "node build/npm/update-all-grammars.js"
"update-grammars": "node build/npm/update-all-grammars.js",
"smoketest": "node test/smoke/out/main.js"
},
"dependencies": {
"applicationinsights": "0.17.1",

27
test/smoke/README.md
View file

@ -1,9 +1,14 @@
# VS Code Smoke Testing
- Run `npm install`
- Start the tests: `npm test -- --latest "path/to/binary"`.
```
npm rum smoketest -- "path/to/code"
```
If you want to include 'Data Migration' area tests use `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively.
If you want to include 'Data Migration' area tests use:
```
npm run smoketest -- "path/to/code" "path/to/codeStable"
```
Detailed prerequisites and running steps are described [in our smoke test wiki](https://github.com/Microsoft/vscode/wiki/Smoke-Test#automated-smoke-test).
@ -50,15 +55,7 @@ To add new test, `./test/${area}.ts` should be updated. The same instruction-sty
Almost on every automated test action it captures a screenshot. These help to determine an issue, if smoke test fails. The normal workflow is that you understand what code is doing and then try to match it up with screenshots obtained from the test.
# Running "Out of Sources"
If you did a fix in VS Code that you need in order for the smoke test to succeed, here is how you can run the smoke test against the sources of VS Code:
* Set related environment variables in the console:
* `export NODE_ENV=development`
* `export VSCODE_DEV=1`
* `export VSCODE_CLI=1`
* open `application.ts`
* pass in the vscode folder as argument to the application
* e.g. instead of `args: args` type `args: ['/Users/bpasero/Development/vscode', ...args]`
* `cd test/smoke`
* `npm install`
* `npm test -- --latest <path to electron>`
* e.g. on macOS: `npm test -- --latest <path to vscode>/.build/electron/Code\ -\ OSS.app/Contents/MacOS/Electron`
```
npm run smoketest
```

22
test/smoke/package.json
View file

@ -3,24 +3,22 @@
"version": "0.1.0",
"main": "./src/main.js",
"scripts": {
"compile": "tsc",
"pretest": "tsc",
"test": "node out/main.js"
"postinstall": "tsc",
"watch": "tsc --watch"
},
"devDependencies": {
"@types/mocha": "^2.2.41",
"@types/node": "^6.0.70",
"@types/webdriverio": "^4.6.1",
"@types/electron": "~1.4.37",
"@types/rimraf": "^0.0.28",
"@types/htmlparser2": "^3.7.29",
"@types/mkdirp": "^0.5.1",
"@types/mocha": "^2.2.41",
"@types/node": "^8.0.26",
"@types/rimraf": "^0.0.28",
"@types/webdriverio": "^4.6.1",
"htmlparser2": "^3.9.2",
"mocha": "^3.2.0",
"spectron": "~3.6.4",
"typescript": "^2.2.2",
"rimraf": "^2.6.1",
"commander": "^2.9.0",
"simple-git": "^1.73.0",
"spectron": "~3.6.4",
"strip-json-comments": "^2.0.1",
"htmlparser2": "^3.9.2"
"typescript": "^2.2.2"
}
}

300
test/smoke/src/main.ts
View file

@ -3,220 +3,132 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
const fs = require('fs');
const https = require('https');
const program = require('commander');
const git = require('simple-git')();
const child_process = require('child_process');
const path = require('path');
const mkdirp = require('mkdirp');
import * as fs from 'fs';
import * as https from 'https';
import * as cp from 'child_process';
import * as path from 'path';
import * as mkdirp from 'mkdirp';
const testDataPath = path.join(process.cwd(), 'test_data');
const codeWorkspacePath = path.join(testDataPath, 'smoketest.code-workspace');
const testDataPath = path.join(__dirname, '..', 'test_data');
const workspacePath = path.join(testDataPath, 'smoketest.code-workspace');
const testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express';
const testRepoLocalDir = path.join(testDataPath, 'vscode-smoketest-express');
const keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings';
mkdirp.sync(testDataPath);
program
.option('-l, --latest <file path>', 'path to the latest VS Code to test')
.option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests');
program.on('--help', () => {
console.log(' Examples:');
console.log('');
console.log(' $ npm test -- --latest path/to/binary');
console.log(' $ npm test -- -l path/to/binary');
console.log('');
console.log(' $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary');
console.log(' $ npm test -- -l path/to/latest/binary -s path/to/stable/binary');
console.log('');
});
program.parse(process.argv);
if (!program.latest) {
fail('You must specify the binary to run the smoke test against');
}
if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) {
fail('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.');
}
if (parseInt(process.version.substr(1)) < 6) {
fail('Please update your Node version to greater than 6 to run the smoke test.');
}
// Setting up environment variables
process.env.VSCODE_LATEST_PATH = program.latest;
if (program.stable) {
process.env.VSCODE_STABLE_PATH = program.stable;
}
process.env.SMOKETEST_REPO = testRepoLocalDir;
if (program.latest && (program.latest.indexOf('Code - Insiders') /* macOS/Windows */ || program.latest.indexOf('code-insiders') /* Linux */) >= 0) {
process.env.VSCODE_EDITION = 'insiders';
}
process.env.VSCODE_WORKSPACE_PATH = codeWorkspacePath;
// Setting up 'vscode-smoketest-express' project
let os = process.platform.toString();
if (os === 'darwin') {
os = 'osx';
}
else if (os === 'win32') {
os = 'win';
}
main().catch(err => console.error(err));
async function main(): Promise<void> {
await getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, path.join(testDataPath, 'keybindings.json'));
const workspace = {
id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
folders: [
toUri(path.join(testRepoLocalDir, 'public')),
toUri(path.join(testRepoLocalDir, 'routes')),
toUri(path.join(testRepoLocalDir, 'views'))
]
};
await createWorkspaceFile(codeWorkspacePath, workspace);
await cleanOrClone(testRepoUrl, testRepoLocalDir);
await execute('npm install', testRepoLocalDir);
await runTests();
}
function fail(errorMessage): void {
console.error(errorMessage);
process.exit(1);
}
if (parseInt(process.version.substr(1)) < 6) {
fail('Please update your Node version to greater than 6 to run the smoke test.');
}
const repoPath = path.join(__dirname, '..', '..', '..');
function getDevElectronPath(): string {
const buildPath = path.join(repoPath, '.build');
const product = require(path.join(repoPath, 'product.json'));
switch (process.platform) {
case 'darwin':
return path.join(buildPath, `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
case 'linux':
return path.join(buildPath, 'electron', `${product.applicationName}`);
case 'win32':
return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
default:
throw new Error('Unsupported platform.');
}
}
let [, , testCodePath, stableCodePath] = process.argv;
if (testCodePath) {
process.env.VSCODE_PATH = testCodePath;
if (stableCodePath) {
process.env.VSCODE_STABLE_PATH = stableCodePath;
}
} else {
testCodePath = getDevElectronPath();
process.env.VSCODE_PATH = testCodePath;
process.env.VSCODE_REPOSITORY = repoPath;
process.env.VSCODE_DEV = '1';
process.env.VSCODE_CLI = '1';
}
if (!fs.existsSync(testCodePath)) {
fail(`Can't find Code at ${testCodePath}.`);
}
process.env.SMOKETEST_REPO = testRepoLocalDir;
process.env.VSCODE_WORKSPACE_PATH = workspacePath;
if ((testCodePath.indexOf('Code - Insiders') /* macOS/Windows */ || testCodePath.indexOf('code-insiders') /* Linux */) >= 0) {
process.env.VSCODE_EDITION = 'insiders';
}
function getKeybindingPlatform(): string {
switch (process.platform) {
case 'darwin': return 'osx';
case 'win32': return 'win';
default: return process.platform;
}
}
function toUri(path: string): string {
if (os === 'win') {
if (process.platform === 'win32') {
return `file:///${path.replace(/\\/g, '/')}`;
}
return `file://${path}`;
}
function runTests(): void {
console.log('Running tests...');
var proc = child_process.spawn(process.execPath, [
'out/mocha-runner.js'
]);
proc.stdout.on('data', data => {
console.log(data.toString());
});
proc.stderr.on('data', data => {
var date = new Date().toLocaleString();
fs.appendFile(path.join(testDataPath, 'errors.log'), `${date}: ${data.toString()}`, (err) => {
if (err) {
throw new Error(`Could not write stderr to errors.log with the following error: ${err}`);
};
});
});
proc.on('exit', (code) => {
process.exit(code);
});
}
async function main(): Promise<void> {
const keybindingsUrl = `https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.${getKeybindingPlatform()}.json`;
console.log(`Fetching keybindings from ${keybindingsUrl}...`);
async function cleanOrClone(repo: string, dir: string): Promise<any> {
console.log('Cleaning or cloning test project repository...');
await new Promise((c, e) => {
https.get(keybindingsUrl, res => {
const output = fs.createWriteStream(path.join(testDataPath, 'keybindings.json'));
res.on('error', e);
output.on('error', e);
output.on('close', c);
res.pipe(output);
}).on('error', e);
});
if (!folderExists(dir)) {
await gitClone(repo, dir);
if (!fs.existsSync(workspacePath)) {
console.log('Creating workspace file...');
const workspace = {
id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
folders: [
toUri(path.join(testRepoLocalDir, 'public')),
toUri(path.join(testRepoLocalDir, 'routes')),
toUri(path.join(testRepoLocalDir, 'views'))
]
};
fs.writeFileSync(workspacePath, JSON.stringify(workspace, null, '\t'));
}
if (!fs.existsSync(testRepoLocalDir)) {
console.log('Cloning test project repository...');
cp.spawnSync('git', ['clone', testRepoUrl, testRepoLocalDir]);
} else {
git.cwd(dir);
await new Promise((c, e) => git.fetch(err => err ? e(err) : c()));
await gitResetAndClean();
console.log('Cleaning test project repository...');
cp.spawnSync('git', ['fetch'], { cwd: testRepoLocalDir });
cp.spawnSync('git', ['reset', '--hard', 'FETCH_HEAD'], { cwd: testRepoLocalDir });
cp.spawnSync('git', ['clean', '-xdf'], { cwd: testRepoLocalDir });
}
console.log('Running npm install...');
cp.execSync('npm install', { cwd: testRepoLocalDir, stdio: 'inherit' });
console.log('Running tests...');
const mocha = cp.spawnSync(process.execPath, ['out/mocha-runner.js'], { cwd: path.join(__dirname, '..'), stdio: 'inherit' });
process.exit(mocha.status);
}
function gitClone(repo: string, dir: string): Promise<any> {
return new Promise((res, rej) => {
git.clone(repo, dir, () => {
console.log('Test repository successfully cloned.');
res();
});
});
}
async function gitResetAndClean(): Promise<any> {
await new Promise((c, e) => git.reset(['FETCH_HEAD', '--hard'], err => err ? e(err) : c()));
await new Promise((c, e) => git.clean('f', ['-d'], err => err ? e(err) : c()));
console.log('Test project was successfully reset to initial state.');
}
function execute(cmd: string, dir: string): Promise<any> {
return new Promise((res, rej) => {
console.log(`Running ${cmd}...`);
child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => {
if (error) {
rej(error);
}
if (stderr) {
console.error(stderr);
}
console.log(stdout);
res();
});
});
}
function getKeybindings(url: string, location: string): Promise<any> {
console.log(`Fetching keybindings from ${url}...`);
return new Promise((resolve, reject) => {
https.get(url, (res) => {
if (res.statusCode !== 200) {
reject(`Failed to obtain key bindings with response code: ${res.statusCode}`);
}
var buffer: Buffer[] = [];
res.on('data', (chunk) => buffer.push(chunk));
res.on('end', () => {
fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => {
console.log('Keybindings were successfully fetched.');
resolve();
});
});
}).on('error', (e) => {
reject(`Failed to obtain key bindings with an error: ${e}`);
});
});
}
function createWorkspaceFile(path: string, workspace: any): Promise<any> {
console.log(`Creating workspace file at ${path}...`);
return new Promise((resolve, reject) => {
fs.exists(path, exists => {
if (exists) {
return resolve();
}
fs.writeFile(path, JSON.stringify(workspace, null, '\t'), error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
});
}
function folderExists(folder: string): boolean {
try {
fs.accessSync(folder, 'rw');
return true;
} catch (e) {
return false;
}
}
function binaryExists(filePath: string): boolean {
try {
fs.accessSync(filePath, 'x');
return true;
} catch (e) {
return false;
}
}
main().catch(fail);

3
test/smoke/src/mocha-runner.ts
View file

@ -4,13 +4,14 @@
*--------------------------------------------------------------------------------------------*/
const MochaTest = require('mocha');
const path = require('path');
const mochaTest = new MochaTest({
timeout: 60000,
slow: 10000,
useColors: true
});
mochaTest.addFile(require('path').join(process.cwd(), 'out/test.js'));
mochaTest.addFile(path.join(__dirname, 'test.js'));
mochaTest.run((failures) => {
process.exit(failures);
});

15
test/smoke/src/spectron/application.ts
View file

@ -9,10 +9,10 @@ import { Screenshot } from '../helpers/screenshot';
var fs = require('fs');
var path = require('path');
export const LATEST_PATH = process.env.VSCODE_LATEST_PATH;
export const STABLE_PATH = process.env.VSCODE_STABLE_PATH;
export const WORKSPACE_PATH = process.env.SMOKETEST_REPO;
export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH;
export const LATEST_PATH = process.env.VSCODE_PATH || '';
export const STABLE_PATH = process.env.VSCODE_STABLE_PATH || '';
export const WORKSPACE_PATH = process.env.SMOKETEST_REPO || '';
export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH || '';
export const USER_DIR = 'test_data/temp_user_dir';
export const EXTENSIONS_DIR = 'test_data/temp_extensions_dir';
@ -50,6 +50,11 @@ export class SpectronApplication {
args.push(`--extensions-dir=${this.sampleExtensionsDir}`);
}
const repo = process.env.VSCODE_REPOSITORY;
if (repo) {
args = [repo, ...args];
}
this.spectron = new Application({
path: electronPath,
args: args,
@ -96,7 +101,7 @@ export class SpectronApplication {
}
private retrieveKeybindings() {
fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => {
fs.readFile(path.join(__dirname, '../../test_data/keybindings.json'), 'utf8', (err, data) => {
if (err) {
throw err;
}

1
test_data/vscode-smoketest-express Submodule

@ -0,0 +1 @@
Subproject commit 5ac7ff24e8c4028e00d8b7d093a920db31513d2e