vscode/build/lib/tsb/builder.ts

686 lines
20 KiB
TypeScript

/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import * as utils from './utils';
import * as colors from 'ansi-colors';
import * as ts from 'typescript';
import * as Vinyl from 'vinyl';
import { RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map';
export interface IConfiguration {
logFn: (topic: string, message: string) => void;
_emitWithoutBasePath?: boolean;
}
export interface CancellationToken {
isCancellationRequested(): boolean;
}
export namespace CancellationToken {
export const None: CancellationToken = {
isCancellationRequested() { return false; }
};
}
export interface ITypeScriptBuilder {
build(out: (file: Vinyl) => void, onError: (err: ts.Diagnostic) => void, token?: CancellationToken): Promise<any>;
file(file: Vinyl): void;
languageService: ts.LanguageService;
}
function normalize(path: string): string {
return path.replace(/\\/g, '/');
}
export function createTypeScriptBuilder(config: IConfiguration, projectFile: string, cmd: ts.ParsedCommandLine): ITypeScriptBuilder {
const _log = config.logFn;
const host = new LanguageServiceHost(cmd, projectFile, _log);
const service = ts.createLanguageService(host, ts.createDocumentRegistry());
const lastBuildVersion: { [path: string]: string } = Object.create(null);
const lastDtsHash: { [path: string]: string } = Object.create(null);
const userWantsDeclarations = cmd.options.declaration;
let oldErrors: { [path: string]: ts.Diagnostic[] } = Object.create(null);
let headUsed = process.memoryUsage().heapUsed;
let emitSourceMapsInStream = true;
// always emit declaraction files
host.getCompilationSettings().declaration = true;
function file(file: Vinyl): void {
// support gulp-sourcemaps
if ((<any>file).sourceMap) {
emitSourceMapsInStream = false;
}
if (!file.contents) {
host.removeScriptSnapshot(file.path);
} else {
host.addScriptSnapshot(file.path, new VinylScriptSnapshot(file));
}
}
function baseFor(snapshot: ScriptSnapshot): string {
if (snapshot instanceof VinylScriptSnapshot) {
return cmd.options.outDir || snapshot.getBase();
} else {
return '';
}
}
function isExternalModule(sourceFile: ts.SourceFile): boolean {
return (<any>sourceFile).externalModuleIndicator
|| /declare\s+module\s+('|")(.+)\1/.test(sourceFile.getText());
}
function build(out: (file: Vinyl) => void, onError: (err: any) => void, token = CancellationToken.None): Promise<any> {
function checkSyntaxSoon(fileName: string): Promise<ts.Diagnostic[]> {
return new Promise<ts.Diagnostic[]>(resolve => {
process.nextTick(function () {
if (!host.getScriptSnapshot(fileName, false)) {
resolve([]); // no script, no problems
} else {
resolve(service.getSyntacticDiagnostics(fileName));
}
});
});
}
function checkSemanticsSoon(fileName: string): Promise<ts.Diagnostic[]> {
return new Promise<ts.Diagnostic[]>(resolve => {
process.nextTick(function () {
if (!host.getScriptSnapshot(fileName, false)) {
resolve([]); // no script, no problems
} else {
resolve(service.getSemanticDiagnostics(fileName));
}
});
});
}
function emitSoon(fileName: string): Promise<{ fileName: string; signature?: string; files: Vinyl[] }> {
return new Promise(resolve => {
process.nextTick(function () {
if (/\.d\.ts$/.test(fileName)) {
// if it's already a d.ts file just emit it signature
const snapshot = host.getScriptSnapshot(fileName);
const signature = crypto.createHash('md5')
.update(snapshot.getText(0, snapshot.getLength()))
.digest('base64');
return resolve({
fileName,
signature,
files: []
});
}
const output = service.getEmitOutput(fileName);
const files: Vinyl[] = [];
let signature: string | undefined;
for (const file of output.outputFiles) {
if (!emitSourceMapsInStream && /\.js\.map$/.test(file.name)) {
continue;
}
if (/\.d\.ts$/.test(file.name)) {
signature = crypto.createHash('md5')
.update(file.text)
.digest('base64');
if (!userWantsDeclarations) {
// don't leak .d.ts files if users don't want them
continue;
}
}
const vinyl = new Vinyl({
path: file.name,
contents: Buffer.from(file.text),
base: !config._emitWithoutBasePath && baseFor(host.getScriptSnapshot(fileName)) || undefined
});
if (!emitSourceMapsInStream && /\.js$/.test(file.name)) {
const sourcemapFile = output.outputFiles.filter(f => /\.js\.map$/.test(f.name))[0];
if (sourcemapFile) {
const extname = path.extname(vinyl.relative);
const basename = path.basename(vinyl.relative, extname);
const dirname = path.dirname(vinyl.relative);
const tsname = (dirname === '.' ? '' : dirname + '/') + basename + '.ts';
let sourceMap = <RawSourceMap>JSON.parse(sourcemapFile.text);
sourceMap.sources[0] = tsname.replace(/\\/g, '/');
// check for an "input source" map and combine them
// in step 1 we extract all line edit from the input source map, and
// in step 2 we apply the line edits to the typescript source map
const snapshot = host.getScriptSnapshot(fileName);
if (snapshot instanceof VinylScriptSnapshot && snapshot.sourceMap) {
const inputSMC = new SourceMapConsumer(snapshot.sourceMap);
const tsSMC = new SourceMapConsumer(sourceMap);
let didChange = false;
const smg = new SourceMapGenerator({
file: sourceMap.file,
sourceRoot: sourceMap.sourceRoot
});
// step 1
const lineEdits = new Map<number, [from: number, to: number][]>();
inputSMC.eachMapping(m => {
if (m.originalLine === m.generatedLine) {
// same line mapping
let array = lineEdits.get(m.originalLine);
if (!array) {
array = [];
lineEdits.set(m.originalLine, array);
}
array.push([m.originalColumn, m.generatedColumn]);
} else {
// NOT SUPPORTED
}
});
// step 2
tsSMC.eachMapping(m => {
didChange = true;
const edits = lineEdits.get(m.originalLine);
let originalColumnDelta = 0;
if (edits) {
for (const [from, to] of edits) {
if (to >= m.originalColumn) {
break;
}
originalColumnDelta = from - to;
}
}
smg.addMapping({
source: m.source,
name: m.name,
generated: { line: m.generatedLine, column: m.generatedColumn },
original: { line: m.originalLine, column: m.originalColumn + originalColumnDelta }
});
});
if (didChange) {
[tsSMC, inputSMC].forEach((consumer) => {
(<SourceMapConsumer & { sources: string[] }>consumer).sources.forEach((sourceFile: any) => {
(<any>smg)._sources.add(sourceFile);
const sourceContent = consumer.sourceContentFor(sourceFile);
if (sourceContent !== null) {
smg.setSourceContent(sourceFile, sourceContent);
}
});
});
sourceMap = JSON.parse(smg.toString());
// const filename = '/Users/jrieken/Code/vscode/src2/' + vinyl.relative + '.map';
// fs.promises.mkdir(path.dirname(filename), { recursive: true }).then(async () => {
// await fs.promises.writeFile(filename, smg.toString());
// await fs.promises.writeFile('/Users/jrieken/Code/vscode/src2/' + vinyl.relative, vinyl.contents);
// });
}
}
(<any>vinyl).sourceMap = sourceMap;
}
}
files.push(vinyl);
}
resolve({
fileName,
signature,
files
});
});
});
}
const newErrors: { [path: string]: ts.Diagnostic[] } = Object.create(null);
const t1 = Date.now();
const toBeEmitted: string[] = [];
const toBeCheckedSyntactically: string[] = [];
const toBeCheckedSemantically: string[] = [];
const filesWithChangedSignature: string[] = [];
const dependentFiles: string[] = [];
const newLastBuildVersion = new Map<string, string>();
for (const fileName of host.getScriptFileNames()) {
if (lastBuildVersion[fileName] !== host.getScriptVersion(fileName)) {
toBeEmitted.push(fileName);
toBeCheckedSyntactically.push(fileName);
toBeCheckedSemantically.push(fileName);
}
}
return new Promise<void>(resolve => {
const semanticCheckInfo = new Map<string, number>();
const seenAsDependentFile = new Set<string>();
function workOnNext() {
let promise: Promise<any> | undefined;
// let fileName: string;
// someone told us to stop this
if (token.isCancellationRequested()) {
_log('[CANCEL]', '>>This compile run was cancelled<<');
newLastBuildVersion.clear();
resolve();
return;
}
// (1st) emit code
else if (toBeEmitted.length) {
const fileName = toBeEmitted.pop()!;
promise = emitSoon(fileName).then(value => {
for (const file of value.files) {
_log('[emit code]', file.path);
out(file);
}
// remember when this was build
newLastBuildVersion.set(fileName, host.getScriptVersion(fileName));
// remeber the signature
if (value.signature && lastDtsHash[fileName] !== value.signature) {
lastDtsHash[fileName] = value.signature;
filesWithChangedSignature.push(fileName);
}
}).catch(e => {
// can't just skip this or make a result up..
host.error(`ERROR emitting ${fileName}`);
host.error(e);
});
}
// (2nd) check syntax
else if (toBeCheckedSyntactically.length) {
const fileName = toBeCheckedSyntactically.pop()!;
_log('[check syntax]', fileName);
promise = checkSyntaxSoon(fileName).then(diagnostics => {
delete oldErrors[fileName];
if (diagnostics.length > 0) {
diagnostics.forEach(d => onError(d));
newErrors[fileName] = diagnostics;
// stop the world when there are syntax errors
toBeCheckedSyntactically.length = 0;
toBeCheckedSemantically.length = 0;
filesWithChangedSignature.length = 0;
}
});
}
// (3rd) check semantics
else if (toBeCheckedSemantically.length) {
let fileName = toBeCheckedSemantically.pop();
while (fileName && semanticCheckInfo.has(fileName)) {
fileName = toBeCheckedSemantically.pop()!;
}
if (fileName) {
_log('[check semantics]', fileName);
promise = checkSemanticsSoon(fileName).then(diagnostics => {
delete oldErrors[fileName!];
semanticCheckInfo.set(fileName!, diagnostics.length);
if (diagnostics.length > 0) {
diagnostics.forEach(d => onError(d));
newErrors[fileName!] = diagnostics;
}
});
}
}
// (4th) check dependents
else if (filesWithChangedSignature.length) {
while (filesWithChangedSignature.length) {
const fileName = filesWithChangedSignature.pop()!;
if (!isExternalModule(service.getProgram()!.getSourceFile(fileName)!)) {
_log('[check semantics*]', fileName + ' is an internal module and it has changed shape -> check whatever hasn\'t been checked yet');
toBeCheckedSemantically.push(...host.getScriptFileNames());
filesWithChangedSignature.length = 0;
dependentFiles.length = 0;
break;
}
host.collectDependents(fileName, dependentFiles);
}
}
// (5th) dependents contd
else if (dependentFiles.length) {
let fileName = dependentFiles.pop();
while (fileName && seenAsDependentFile.has(fileName)) {
fileName = dependentFiles.pop();
}
if (fileName) {
seenAsDependentFile.add(fileName);
const value = semanticCheckInfo.get(fileName);
if (value === 0) {
// already validated successfully -> look at dependents next
host.collectDependents(fileName, dependentFiles);
} else if (typeof value === 'undefined') {
// first validate -> look at dependents next
dependentFiles.push(fileName);
toBeCheckedSemantically.push(fileName);
}
}
}
// (last) done
else {
resolve();
return;
}
if (!promise) {
promise = Promise.resolve();
}
promise.then(function () {
// change to change
process.nextTick(workOnNext);
}).catch(err => {
console.error(err);
});
}
workOnNext();
}).then(() => {
// store the build versions to not rebuilt the next time
newLastBuildVersion.forEach((value, key) => {
lastBuildVersion[key] = value;
});
// print old errors and keep them
utils.collections.forEach(oldErrors, entry => {
entry.value.forEach(diag => onError(diag));
newErrors[entry.key] = entry.value;
});
oldErrors = newErrors;
// print stats
const headNow = process.memoryUsage().heapUsed;
const MB = 1024 * 1024;
_log(
'[tsb]',
`time: ${colors.yellow((Date.now() - t1) + 'ms')} + \nmem: ${colors.cyan(Math.ceil(headNow / MB) + 'MB')} ${colors.bgCyan('delta: ' + Math.ceil((headNow - headUsed) / MB))}`
);
headUsed = headNow;
});
}
return {
file,
build,
languageService: service
};
}
class ScriptSnapshot implements ts.IScriptSnapshot {
private readonly _text: string;
private readonly _mtime: Date;
constructor(text: string, mtime: Date) {
this._text = text;
this._mtime = mtime;
}
getVersion(): string {
return this._mtime.toUTCString();
}
getText(start: number, end: number): string {
return this._text.substring(start, end);
}
getLength(): number {
return this._text.length;
}
getChangeRange(_oldSnapshot: ts.IScriptSnapshot): ts.TextChangeRange | undefined {
return undefined;
}
}
class VinylScriptSnapshot extends ScriptSnapshot {
private readonly _base: string;
readonly sourceMap?: RawSourceMap;
constructor(file: Vinyl & { sourceMap?: RawSourceMap }) {
super(file.contents!.toString(), file.stat!.mtime);
this._base = file.base;
this.sourceMap = file.sourceMap;
}
getBase(): string {
return this._base;
}
}
class LanguageServiceHost implements ts.LanguageServiceHost {
private readonly _snapshots: { [path: string]: ScriptSnapshot };
private readonly _filesInProject: Set<string>;
private readonly _filesAdded: Set<string>;
private readonly _dependencies: utils.graph.Graph<string>;
private readonly _dependenciesRecomputeList: string[];
private readonly _fileNameToDeclaredModule: { [path: string]: string[] };
private _projectVersion: number;
constructor(
private readonly _cmdLine: ts.ParsedCommandLine,
private readonly _projectPath: string,
private readonly _log: (topic: string, message: string) => void
) {
this._snapshots = Object.create(null);
this._filesInProject = new Set(_cmdLine.fileNames);
this._filesAdded = new Set();
this._dependencies = new utils.graph.Graph<string>(s => s);
this._dependenciesRecomputeList = [];
this._fileNameToDeclaredModule = Object.create(null);
this._projectVersion = 1;
}
log(_s: string): void {
// console.log(s);
}
trace(_s: string): void {
// console.log(s);
}
error(s: string): void {
console.error(s);
}
getCompilationSettings(): ts.CompilerOptions {
return this._cmdLine.options;
}
getProjectVersion(): string {
return String(this._projectVersion);
}
getScriptFileNames(): string[] {
const res = Object.keys(this._snapshots).filter(path => this._filesInProject.has(path) || this._filesAdded.has(path));
return res;
}
getScriptVersion(filename: string): string {
filename = normalize(filename);
const result = this._snapshots[filename];
if (result) {
return result.getVersion();
}
return 'UNKNWON_FILE_' + Math.random().toString(16).slice(2);
}
getScriptSnapshot(filename: string, resolve: boolean = true): ScriptSnapshot {
filename = normalize(filename);
let result = this._snapshots[filename];
if (!result && resolve) {
try {
result = new VinylScriptSnapshot(new Vinyl(<any>{
path: filename,
contents: fs.readFileSync(filename),
base: this.getCompilationSettings().outDir,
stat: fs.statSync(filename)
}));
this.addScriptSnapshot(filename, result);
} catch (e) {
// ignore
}
}
return result;
}
private static _declareModule = /declare\s+module\s+('|")(.+)\1/g;
addScriptSnapshot(filename: string, snapshot: ScriptSnapshot): ScriptSnapshot {
this._projectVersion++;
filename = normalize(filename);
const old = this._snapshots[filename];
if (!old && !this._filesInProject.has(filename) && !filename.endsWith('.d.ts')) {
// ^^^^^^^^^^^^^^^^^^^^^^^^^^
// not very proper!
this._filesAdded.add(filename);
}
if (!old || old.getVersion() !== snapshot.getVersion()) {
this._dependenciesRecomputeList.push(filename);
const node = this._dependencies.lookup(filename);
if (node) {
node.outgoing = Object.create(null);
}
// (cheap) check for declare module
LanguageServiceHost._declareModule.lastIndex = 0;
let match: RegExpExecArray | null | undefined;
while ((match = LanguageServiceHost._declareModule.exec(snapshot.getText(0, snapshot.getLength())))) {
let declaredModules = this._fileNameToDeclaredModule[filename];
if (!declaredModules) {
this._fileNameToDeclaredModule[filename] = declaredModules = [];
}
declaredModules.push(match[2]);
}
}
this._snapshots[filename] = snapshot;
return old;
}
removeScriptSnapshot(filename: string): boolean {
this._filesInProject.delete(filename);
this._filesAdded.delete(filename);
this._projectVersion++;
filename = normalize(filename);
delete this._fileNameToDeclaredModule[filename];
return delete this._snapshots[filename];
}
getCurrentDirectory(): string {
return path.dirname(this._projectPath);
}
getDefaultLibFileName(options: ts.CompilerOptions): string {
return ts.getDefaultLibFilePath(options);
}
readonly directoryExists = ts.sys.directoryExists;
readonly getDirectories = ts.sys.getDirectories;
readonly fileExists = ts.sys.fileExists;
readonly readFile = ts.sys.readFile;
readonly readDirectory = ts.sys.readDirectory;
// ---- dependency management
collectDependents(filename: string, target: string[]): void {
while (this._dependenciesRecomputeList.length) {
this._processFile(this._dependenciesRecomputeList.pop()!);
}
filename = normalize(filename);
const node = this._dependencies.lookup(filename);
if (node) {
utils.collections.forEach(node.incoming, entry => target.push(entry.key));
}
}
_processFile(filename: string): void {
if (filename.match(/.*\.d\.ts$/)) {
return;
}
filename = normalize(filename);
const snapshot = this.getScriptSnapshot(filename);
if (!snapshot) {
this._log('processFile', `Missing snapshot for: ${filename}`);
return;
}
const info = ts.preProcessFile(snapshot.getText(0, snapshot.getLength()), true);
// (1) ///-references
info.referencedFiles.forEach(ref => {
const resolvedPath = path.resolve(path.dirname(filename), ref.fileName);
const normalizedPath = normalize(resolvedPath);
this._dependencies.inertEdge(filename, normalizedPath);
});
// (2) import-require statements
info.importedFiles.forEach(ref => {
const stopDirname = normalize(this.getCurrentDirectory());
let dirname = filename;
let found = false;
while (!found && dirname.indexOf(stopDirname) === 0) {
dirname = path.dirname(dirname);
const resolvedPath = path.resolve(dirname, ref.fileName);
const normalizedPath = normalize(resolvedPath);
if (this.getScriptSnapshot(normalizedPath + '.ts')) {
this._dependencies.inertEdge(filename, normalizedPath + '.ts');
found = true;
} else if (this.getScriptSnapshot(normalizedPath + '.d.ts')) {
this._dependencies.inertEdge(filename, normalizedPath + '.d.ts');
found = true;
}
}
if (!found) {
for (const key in this._fileNameToDeclaredModule) {
if (this._fileNameToDeclaredModule[key] && ~this._fileNameToDeclaredModule[key].indexOf(ref.fileName)) {
this._dependencies.inertEdge(filename, key);
}
}
}
});
}
}