Two-pass module evaluation.

Plus changes to tests to accomodate.
This commit is contained in:
Kitson Kelly 2018-08-27 21:22:57 -07:00 committed by Ryan Dahl
parent 9960b1d22b
commit 54aefa2886
5 changed files with 330 additions and 210 deletions

View file

@ -18,7 +18,12 @@ type AmdCallback = (...args: any[]) => void;
type AmdErrback = (err: any) => void;
export type AmdFactory = (...args: any[]) => object | void;
// tslint:enable:no-any
export type AmdDefine = (deps: string[], factory: AmdFactory) => void;
export type AmdDefine = (deps: ModuleSpecifier[], factory: AmdFactory) => void;
type AMDRequire = (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
) => void;
// The location that a module is being loaded from. This could be a directory,
// like ".", or it could be a module specifier like
@ -60,7 +65,10 @@ export interface Ts {
* the module, not the actual module instance.
*/
export class ModuleMetaData implements ts.IScriptSnapshot {
public deps?: ModuleFileName[];
public readonly exports = {};
public factory?: AmdFactory;
public hasRun = false;
public scriptVersion = "";
constructor(
@ -155,6 +163,9 @@ export class DenoCompiler implements ts.LanguageServiceHost {
// A reference to the `./os.ts` module, so it can be monkey patched during
// testing
private _os: Os = os;
// Contains a queue of modules that have been resolved, but not yet
// run
private _runQueue: ModuleMetaData[] = [];
// Used to contain the script file we are currently running
private _scriptFileNames: string[] = [];
// A reference to the TypeScript LanguageService instance so it can be
@ -167,6 +178,78 @@ export class DenoCompiler implements ts.LanguageServiceHost {
// testing
private _window = window;
/**
* Drain the run queue, retrieving the arguments for the module
* factory and calling the module's factory.
*/
private _drainRunQueue(): void {
this._log(
"compiler._drainRunQueue",
this._runQueue.map(metaData => metaData.fileName)
);
let moduleMetaData: ModuleMetaData | undefined;
while ((moduleMetaData = this._runQueue.shift())) {
assert(
moduleMetaData.factory != null,
"Cannot run module without factory."
);
assert(moduleMetaData.hasRun === false, "Module has already been run.");
// asserts not tracked by TypeScripts, so using not null operator
moduleMetaData.factory!(...this._getFactoryArguments(moduleMetaData));
moduleMetaData.hasRun = true;
}
}
/**
* Get the dependencies for a given module, but don't run the module,
* just add the module factory to the run queue.
*/
private _gatherDependencies(moduleMetaData: ModuleMetaData): void {
this._log("compiler._resolveDependencies", moduleMetaData.fileName);
// if the module has already run, we can short circuit.
// it is intentional though that if we have already resolved dependencies,
// we won't short circuit, as something may have changed, or we might have
// only collected the dependencies to be able to able to obtain the graph of
// dependencies
if (moduleMetaData.hasRun) {
return;
}
this._window.define = this.makeDefine(moduleMetaData);
this._globalEval(this.compile(moduleMetaData));
this._window.define = undefined;
}
/**
* Retrieve the arguments to pass a module's factory function.
*/
// tslint:disable-next-line:no-any
private _getFactoryArguments(moduleMetaData: ModuleMetaData): any[] {
if (!moduleMetaData.deps) {
throw new Error("Cannot get arguments until dependencies resolved.");
}
return moduleMetaData.deps.map(dep => {
if (dep === "require") {
return this._makeLocalRequire(moduleMetaData);
}
if (dep === "exports") {
return moduleMetaData.exports;
}
if (dep in DenoCompiler._builtins) {
return DenoCompiler._builtins[dep];
}
const dependencyMetaData = this._getModuleMetaData(dep);
assert(dependencyMetaData != null, `Missing dependency "${dep}".`);
assert(
dependencyMetaData!.hasRun === true,
`Module "${dep}" was not run.`
);
// TypeScript does not track assert, therefore using not null operator
return dependencyMetaData!.exports;
});
}
/**
* The TypeScript language service often refers to the resolved fileName of
* a module, this is a shortcut to avoid unnecessary module resolution logic
@ -186,6 +269,35 @@ export class DenoCompiler implements ts.LanguageServiceHost {
: undefined;
}
/**
* Returns a require that specifically handles the resolution of a transpiled
* emit of a dynamic ES `import()` from TypeScript.
*/
private _makeLocalRequire(moduleMetaData: ModuleMetaData): AMDRequire {
const localRequire = (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
): void => {
log("localRequire", deps);
assert(
deps.length === 1,
"Local require requires exactly one dependency."
);
const [moduleSpecifier] = deps;
try {
const requiredMetaData = this.run(
moduleSpecifier,
moduleMetaData.fileName
);
callback(requiredMetaData.exports);
} catch (e) {
errback(e);
}
};
return localRequire;
}
/**
* Setup being able to map back source references back to their source
*
@ -232,7 +344,12 @@ export class DenoCompiler implements ts.LanguageServiceHost {
/**
* Retrieve the output of the TypeScript compiler for a given `fileName`.
*/
compile(fileName: ModuleFileName): OutputCode {
compile(moduleMetaData: ModuleMetaData): OutputCode {
this._log("compiler.compile", moduleMetaData.fileName);
if (moduleMetaData.outputCode) {
return moduleMetaData.outputCode;
}
const { fileName, sourceCode } = moduleMetaData;
const service = this._service;
const output = service.getEmitOutput(fileName);
@ -263,53 +380,72 @@ export class DenoCompiler implements ts.LanguageServiceHost {
);
const [outputFile] = output.outputFiles;
return outputFile.text;
const outputCode = (moduleMetaData.outputCode = `${
outputFile.text
}\n//# sourceURL=${fileName}`);
moduleMetaData.scriptVersion = "1";
this._os.codeCache(fileName, sourceCode, outputCode);
return moduleMetaData.outputCode;
}
/**
* For a given module specifier and containing file, return a list of absolute
* identifiers for dependent modules that are required by this module.
*/
getModuleDependencies(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleFileName[] {
assert(
this._runQueue.length === 0,
"Cannot get dependencies with modules queued to be run."
);
const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile);
assert(
!moduleMetaData.hasRun,
"Cannot get dependencies for a module that has already been run."
);
this._gatherDependencies(moduleMetaData);
const dependencies = this._runQueue.map(
moduleMetaData => moduleMetaData.fileName
);
// empty the run queue, to free up references to factories we have collected
// and to ensure that if there is a further invocation of `.run()` the
// factories don't get called
this._runQueue = [];
return dependencies;
}
/**
* Create a localized AMD `define` function and return it.
*/
makeDefine(moduleMetaData: ModuleMetaData): AmdDefine {
const localDefine = (deps: string[], factory: AmdFactory): void => {
// TypeScript will emit a local require dependency when doing dynamic
// `import()`
const { _log: log } = this;
const localExports = moduleMetaData.exports;
// tslint:disable-next-line:no-any
const resolveDependencies = (deps: string[]): any[] => {
return deps.map(dep => {
if (dep === "require") {
return localRequire;
} else if (dep === "exports") {
return localExports;
} else if (dep in DenoCompiler._builtins) {
return DenoCompiler._builtins[dep];
} else {
const depModuleMetaData = this.run(dep, moduleMetaData.fileName);
return depModuleMetaData.exports;
}
});
};
// this is a function because we need hoisting
function localRequire(
deps: string[],
callback: AmdCallback,
errback: AmdErrback
): void {
log("localRequire", deps);
try {
const args = resolveDependencies(deps);
callback(...args);
} catch (e) {
errback(e);
// TODO should this really be part of the public API of the compiler?
const localDefine: AmdDefine = (
deps: ModuleSpecifier[],
factory: AmdFactory
): void => {
this._log("compiler.localDefine", moduleMetaData.fileName);
moduleMetaData.factory = factory;
// we will recursively resolve the dependencies for any modules
moduleMetaData.deps = deps.map(dep => {
if (
dep === "require" ||
dep === "exports" ||
dep in DenoCompiler._builtins
) {
return dep;
}
const dependencyMetaData = this.resolveModule(
dep,
moduleMetaData.fileName
);
this._gatherDependencies(dependencyMetaData);
return dependencyMetaData.fileName;
});
if (!this._runQueue.includes(moduleMetaData)) {
this._runQueue.push(moduleMetaData);
}
this._log("localDefine", moduleMetaData.fileName, deps, localExports);
const args = resolveDependencies(deps);
factory(...args);
};
return localDefine;
}
@ -404,32 +540,22 @@ export class DenoCompiler implements ts.LanguageServiceHost {
return moduleMetaData ? moduleMetaData.fileName : undefined;
}
/* tslint:disable-next-line:no-any */
/**
* Execute a module based on the `moduleSpecifier` and the `containingFile`
* and return the resulting `FileModule`.
* Load and run a module and all of its dependencies based on a module
* specifier and a containing file
*/
run(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleMetaData {
this._log("run", { moduleSpecifier, containingFile });
this._log("compiler.run", { moduleSpecifier, containingFile });
const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile);
const fileName = moduleMetaData.fileName;
this._scriptFileNames = [fileName];
const sourceCode = moduleMetaData.sourceCode;
let outputCode = moduleMetaData.outputCode;
if (!outputCode) {
outputCode = moduleMetaData.outputCode = `${this.compile(
fileName
)}\n//# sourceURL=${fileName}`;
moduleMetaData!.scriptVersion = "1";
this._os.codeCache(fileName, sourceCode, outputCode);
this._scriptFileNames = [moduleMetaData.fileName];
if (!moduleMetaData.deps) {
this._gatherDependencies(moduleMetaData);
}
this._window.define = this.makeDefine(moduleMetaData);
this._globalEval(moduleMetaData.outputCode);
this._window.define = undefined;
return moduleMetaData!;
this._drainRunQueue();
return moduleMetaData;
}
/**

View file

@ -6,7 +6,7 @@ import * as ts from "typescript";
// We use a silly amount of `any` in these tests...
// tslint:disable:no-any
const { DenoCompiler, ModuleMetaData } = compiler;
const { DenoCompiler } = compiler;
// Enums like this don't exist at runtime, so local copy
enum ScriptKind {
@ -93,13 +93,7 @@ const moduleMap: {
"foo/baz",
"/root/project/foo/baz.ts",
fooBazTsSource,
null
),
"foo/qat.ts": mockModuleInfo(
"foo/qat",
"/root/project/foo/qat.ts",
null,
null
fooBazTsOutput
)
},
"/root/project/foo/baz.ts": {
@ -108,12 +102,6 @@ const moduleMap: {
"/root/project/foo/bar.ts",
fooBarTsSource,
fooBarTsOutput
),
"./qat.ts": mockModuleInfo(
"foo/qat",
"/root/project/foo/qat.ts",
"export const foo = 'bar'",
null
)
}
};
@ -135,21 +123,13 @@ let codeFetchStack: Array<{
containingFile: string;
}> = [];
function reset() {
codeFetchStack = [];
codeCacheStack = [];
logStack = [];
getEmitOutputStack = [];
globalEvalStack = [];
}
let mockDeps: string[] | undefined;
let mockFactory: compiler.AmdFactory;
let mockDepsStack: string[][] = [];
let mockFactoryStack: compiler.AmdFactory[] = [];
function globalEvalMock(x: string): void {
globalEvalStack.push(x);
if (windowMock.define && mockDeps && mockFactory) {
windowMock.define(mockDeps, mockFactory);
if (windowMock.define && mockDepsStack.length && mockFactoryStack.length) {
windowMock.define(mockDepsStack.pop(), mockFactoryStack.pop());
}
}
function logMock(...args: any[]): void {
@ -231,16 +211,37 @@ const mocks = {
_window: windowMock
};
// Setup the mocks
test(function compilerTestsSetup() {
assert("_globalEval" in compilerInstance);
assert("_log" in compilerInstance);
assert("_os" in compilerInstance);
assert("_ts" in compilerInstance);
assert("_service" in compilerInstance);
assert("_window" in compilerInstance);
/**
* Setup the mocks for a test
*/
function setup() {
// monkey patch mocks on instance
Object.assign(compilerInstance, mocks);
});
}
/**
* Teardown the mocks for a test
*/
function teardown() {
// reset compiler internal state
(compilerInstance as any)._moduleMetaDataMap.clear();
(compilerInstance as any)._fileNamesMap.clear();
// reset mock states
codeFetchStack = [];
codeCacheStack = [];
logStack = [];
getEmitOutputStack = [];
globalEvalStack = [];
assertEqual(mockDepsStack.length, 0);
assertEqual(mockFactoryStack.length, 0);
mockDepsStack = [];
mockFactoryStack = [];
// restore original properties and methods
Object.assign(compilerInstance, originals);
}
test(function compilerInstance() {
assert(DenoCompiler != null);
@ -249,120 +250,105 @@ test(function compilerInstance() {
// Testing the internal APIs
test(function compilerMakeDefine() {
const moduleMetaData = new ModuleMetaData(
"/root/project/foo/bar.ts",
fooBarTsSource,
fooBarTsOutput
);
const localDefine = compilerInstance.makeDefine(moduleMetaData);
let factoryCalled = false;
localDefine(
["require", "exports", "compiler"],
(_require, _exports, _compiler): void => {
factoryCalled = true;
assertEqual(
typeof _require,
"function",
"localRequire should be a function"
);
assert(_exports != null);
assert(
Object.keys(_exports).length === 0,
"exports should have no properties"
);
assert(compiler === _compiler, "compiler should be passed to factory");
}
);
assert(factoryCalled, "Factory expected to be called");
});
// TODO testMakeDefineExternalModule - testing that make define properly runs
// external modules, this is implicitly tested though in
// `compilerRunMultiModule`
test(function compilerLocalRequire() {
const moduleMetaData = new ModuleMetaData(
"/root/project/foo/baz.ts",
fooBazTsSource,
fooBazTsOutput
);
const localDefine = compilerInstance.makeDefine(moduleMetaData);
let requireCallbackCalled = false;
localDefine(
["require", "exports"],
(_require, _exports, _compiler): void => {
assertEqual(typeof _require, "function");
_require(
["./qat.ts"],
_qat => {
requireCallbackCalled = true;
assert(_qat);
},
() => {
throw new Error("Should not error");
}
);
}
);
assert(requireCallbackCalled, "Factory expected to be called");
});
test(function compilerRun() {
// equal to `deno foo/bar.ts`
reset();
const result = compilerInstance.run("foo/bar.ts", "/root/project");
assert(result instanceof ModuleMetaData);
assertEqual(codeFetchStack.length, 1);
assertEqual(codeCacheStack.length, 1);
assertEqual(globalEvalStack.length, 1);
setup();
let factoryRun = false;
mockDepsStack.push(["require", "exports", "compiler"]);
mockFactoryStack.push((_require, _exports, _compiler) => {
factoryRun = true;
assertEqual(typeof _require, "function");
assertEqual(typeof _exports, "object");
assert(_compiler === compiler);
_exports.foo = "bar";
});
const moduleMetaData = compilerInstance.run("foo/bar.ts", "/root/project");
assert(factoryRun);
assert(moduleMetaData.hasRun);
assertEqual(moduleMetaData.sourceCode, fooBarTsSource);
assertEqual(moduleMetaData.outputCode, fooBarTsOutput);
assertEqual(moduleMetaData.exports, { foo: "bar" });
const lastGlobalEval = globalEvalStack.pop();
assertEqual(lastGlobalEval, fooBarTsOutput);
const lastCodeFetch = codeFetchStack.pop();
assertEqual(lastCodeFetch, {
moduleSpecifier: "foo/bar.ts",
containingFile: "/root/project"
});
const lastCodeCache = codeCacheStack.pop();
assertEqual(lastCodeCache, {
fileName: "/root/project/foo/bar.ts",
sourceCode: fooBarTsSource,
outputCode: fooBarTsOutput
});
assertEqual(
codeFetchStack.length,
1,
"Module should have only been fetched once."
);
assertEqual(
codeCacheStack.length,
1,
"Compiled code should have only been cached once."
);
teardown();
});
test(function compilerRunMultiModule() {
// equal to `deno foo/baz.ts`
reset();
let factoryRun = false;
mockDeps = ["require", "exports", "compiler"];
mockFactory = (...deps: any[]) => {
const [_require, _exports, _compiler] = deps;
assertEqual(typeof _require, "function");
assertEqual(typeof _exports, "object");
assertEqual(_compiler, compiler);
factoryRun = true;
Object.defineProperty(_exports, "__esModule", { value: true });
_exports.foo = "bar";
// it is too complicated to test the outer factory, because the localised
// make define already has a reference to this factory and it can't really
// be easily unwound. So we will do what we can with the inner one and
// then just clear it...
mockDeps = undefined;
mockFactory = undefined;
setup();
const factoryStack: string[] = [];
const bazDeps = ["require", "exports", "./bar.ts"];
const bazFactory = (_require, _exports, _bar) => {
factoryStack.push("baz");
assertEqual(_bar.foo, "bar");
};
const barDeps = ["require", "exports", "compiler"];
const barFactory = (_require, _exports, _compiler) => {
factoryStack.push("bar");
_exports.foo = "bar";
};
mockDepsStack.push(barDeps);
mockFactoryStack.push(barFactory);
mockDepsStack.push(bazDeps);
mockFactoryStack.push(bazFactory);
compilerInstance.run("foo/baz.ts", "/root/project");
assertEqual(factoryStack, ["bar", "baz"]);
const result = compilerInstance.run("foo/baz.ts", "/root/project");
assert(result instanceof ModuleMetaData);
// we have mocked that foo/bar.ts is already cached, so two fetches,
// but only a single cache
assertEqual(codeFetchStack.length, 2);
assertEqual(codeCacheStack.length, 1);
// because of the challenges with the way the module factories are generated
// we only get one invocation of the `globalEval` mock.
assertEqual(globalEvalStack.length, 1);
assert(factoryRun);
assertEqual(
codeFetchStack.length,
2,
"Modules should have only been fetched once."
);
assertEqual(codeCacheStack.length, 0, "No code should have been cached.");
teardown();
});
test(function compilerResolveModule() {
setup();
const moduleMetaData = compilerInstance.resolveModule(
"foo/baz.ts",
"/root/project"
);
assertEqual(moduleMetaData.sourceCode, fooBazTsSource);
assertEqual(moduleMetaData.outputCode, fooBazTsOutput);
assert(!moduleMetaData.hasRun);
assert(!moduleMetaData.deps);
assertEqual(moduleMetaData.exports, {});
assertEqual(moduleMetaData.scriptVersion, "1");
assertEqual(codeFetchStack.length, 1, "Only initial module is resolved.");
teardown();
});
test(function compilerGetModuleDependencies() {
setup();
const bazDeps = ["require", "exports", "./bar.ts"];
const bazFactory = () => {
throw new Error("Unexpected factory call");
};
const barDeps = ["require", "exports", "compiler"];
const barFactory = () => {
throw new Error("Unexpected factory call");
};
mockDepsStack.push(barDeps);
mockFactoryStack.push(barFactory);
mockDepsStack.push(bazDeps);
mockFactoryStack.push(bazFactory);
const deps = compilerInstance.getModuleDependencies(
"foo/baz.ts",
"/root/project"
);
assertEqual(deps, ["/root/project/foo/bar.ts", "/root/project/foo/baz.ts"]);
teardown();
});
// TypeScript LanguageServiceHost APIs
@ -388,10 +374,12 @@ test(function compilerGetNewLine() {
});
test(function compilerGetScriptFileNames() {
setup();
compilerInstance.run("foo/bar.ts", "/root/project");
const result = compilerInstance.getScriptFileNames();
assertEqual(result.length, 1, "Expected only a single filename.");
assertEqual(result[0], "/root/project/foo/bar.ts");
teardown();
});
test(function compilerGetScriptKind() {
@ -403,15 +391,18 @@ test(function compilerGetScriptKind() {
});
test(function compilerGetScriptVersion() {
setup();
const moduleMetaData = compilerInstance.resolveModule(
"foo/bar.ts",
"/root/project"
);
compilerInstance.compile(moduleMetaData);
assertEqual(
compilerInstance.getScriptVersion(moduleMetaData.fileName),
"1",
"Expected known module to have script version of 1"
);
teardown();
});
test(function compilerGetScriptVersionUnknown() {
@ -423,6 +414,7 @@ test(function compilerGetScriptVersionUnknown() {
});
test(function compilerGetScriptSnapshot() {
setup();
const moduleMetaData = compilerInstance.resolveModule(
"foo/bar.ts",
"/root/project"
@ -444,6 +436,7 @@ test(function compilerGetScriptSnapshot() {
result === moduleMetaData,
"result should strictly equal moduleMetaData"
);
teardown();
});
test(function compilerGetCurrentDirectory() {
@ -451,10 +444,12 @@ test(function compilerGetCurrentDirectory() {
});
test(function compilerGetDefaultLibFileName() {
setup();
assertEqual(
compilerInstance.getDefaultLibFileName(),
"$asset$/lib.globals.d.ts"
);
teardown();
});
test(function compilerUseCaseSensitiveFileNames() {
@ -473,6 +468,7 @@ test(function compilerReadFile() {
});
test(function compilerFileExists() {
setup();
const moduleMetaData = compilerInstance.resolveModule(
"foo/bar.ts",
"/root/project"
@ -483,9 +479,11 @@ test(function compilerFileExists() {
compilerInstance.fileExists("/root/project/unknown-module.ts"),
false
);
teardown();
});
test(function compilerResolveModuleNames() {
setup();
const results = compilerInstance.resolveModuleNames(
["foo/bar.ts", "foo/baz.ts", "$asset$/lib.globals.d.ts", "deno"],
"/root/project"
@ -503,9 +501,5 @@ test(function compilerResolveModuleNames() {
assertEqual(result.resolvedFileName, resolvedFileName);
assertEqual(result.isExternalLibraryImport, isExternalLibraryImport);
}
});
// Remove the mocks
test(function compilerTestsTeardown() {
Object.assign(compilerInstance, originals);
teardown();
});

View file

@ -4,6 +4,7 @@ Error: error
at foo ([WILDCARD]tests/async_error.ts:4:9)
at eval ([WILDCARD]tests/async_error.ts:7:1)
at DenoCompiler.eval [as _globalEval] (<anonymous>)
at DenoCompiler._gatherDependencies (deno/js/compiler.ts:[WILDCARD])
at DenoCompiler.run (deno/js/compiler.ts:[WILDCARD])
at denoMain (deno/js/main.ts:[WILDCARD])
at deno_main.js:1:1

View file

@ -3,6 +3,7 @@ Error: bad
at bar (file://[WILDCARD]tests/error_001.ts:6:3)
at eval (file://[WILDCARD]tests/error_001.ts:9:1)
at DenoCompiler.eval [as _globalEval] (<anonymous>)
at DenoCompiler._gatherDependencies (deno/js/compiler.ts:[WILDCARD])
at DenoCompiler.run (deno/js/compiler.ts:[WILDCARD])
at denoMain (deno/js/main.ts:[WILDCARD])
at deno_main.js:1:1

View file

@ -1,10 +1,8 @@
Error: exception from mod1
at Object.throwsError (file://[WILDCARD]tests/subdir/mod1.ts:16:9)
at foo (file://[WILDCARD]tests/error_002.ts:4:3)
at eval (file://[WILDCARD]tests/error_002.ts:7:1)
at localDefine (deno/js/compiler.ts:[WILDCARD])
at eval ([WILDCARD]tests/error_002.ts, <anonymous>)
at DenoCompiler.eval [as _globalEval] (<anonymous>)
at Object.throwsError (file://[WILDCARD]deno/tests/subdir/mod1.ts:16:9)
at foo (file://[WILDCARD]deno/tests/error_002.ts:4:3)
at ModuleMetaData.eval [as factory ] (file://[WILDCARD]deno/tests/error_002.ts:7:1)
at DenoCompiler._drainRunQueue (deno/js/compiler.ts:[WILDCARD])
at DenoCompiler.run (deno/js/compiler.ts:[WILDCARD])
at denoMain (deno/js/main.ts:[WILDCARD])
at deno_main.js:1:1