mirror of
https://github.com/denoland/deno
synced 2024-08-25 02:57:22 +00:00
feat: parallelize downloads from TS compiler (#2949)
This commit is contained in:
parent
7e3296dad9
commit
686b86edb1
|
@ -33,39 +33,49 @@ pub fn op_cache(
|
|||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct FetchSourceFileArgs {
|
||||
specifier: String,
|
||||
struct FetchSourceFilesArgs {
|
||||
specifiers: Vec<String>,
|
||||
referrer: String,
|
||||
}
|
||||
|
||||
pub fn op_fetch_source_file(
|
||||
pub fn op_fetch_source_files(
|
||||
state: &ThreadSafeState,
|
||||
args: Value,
|
||||
_zero_copy: Option<PinnedBuf>,
|
||||
) -> Result<JsonOp, ErrBox> {
|
||||
let args: FetchSourceFileArgs = serde_json::from_value(args)?;
|
||||
let args: FetchSourceFilesArgs = serde_json::from_value(args)?;
|
||||
|
||||
// TODO(ry) Maybe a security hole. Only the compiler worker should have access
|
||||
// to this. Need a test to demonstrate the hole.
|
||||
let is_dyn_import = false;
|
||||
|
||||
let resolved_specifier =
|
||||
state.resolve(&args.specifier, &args.referrer, false, is_dyn_import)?;
|
||||
|
||||
let fut = state
|
||||
.file_fetcher
|
||||
.fetch_source_file_async(&resolved_specifier);
|
||||
let mut futures = vec![];
|
||||
for specifier in &args.specifiers {
|
||||
let resolved_specifier =
|
||||
state.resolve(specifier, &args.referrer, false, is_dyn_import)?;
|
||||
let fut = state
|
||||
.file_fetcher
|
||||
.fetch_source_file_async(&resolved_specifier);
|
||||
futures.push(fut);
|
||||
}
|
||||
|
||||
// WARNING: Here we use tokio_util::block_on() which starts a new Tokio
|
||||
// runtime for executing the future. This is so we don't inadvernently run
|
||||
// runtime for executing the future. This is so we don't inadvertently run
|
||||
// out of threads in the main runtime.
|
||||
let out = tokio_util::block_on(fut)?;
|
||||
Ok(JsonOp::Sync(json!({
|
||||
"moduleName": out.url.to_string(),
|
||||
"filename": out.filename.to_str().unwrap(),
|
||||
"mediaType": out.media_type as i32,
|
||||
"sourceCode": String::from_utf8(out.source_code).unwrap(),
|
||||
})))
|
||||
let files = tokio_util::block_on(futures::future::join_all(futures))?;
|
||||
let res: Vec<serde_json::value::Value> = files
|
||||
.into_iter()
|
||||
.map(|file| {
|
||||
json!({
|
||||
"moduleName": file.url.to_string(),
|
||||
"filename": file.filename.to_str().unwrap(),
|
||||
"mediaType": file.media_type as i32,
|
||||
"sourceCode": String::from_utf8(file.source_code).unwrap(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(JsonOp::Sync(json!(res)))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
|
|
@ -37,7 +37,7 @@ pub const OP_START: OpId = 10;
|
|||
pub const OP_APPLY_SOURCE_MAP: OpId = 11;
|
||||
pub const OP_FORMAT_ERROR: OpId = 12;
|
||||
pub const OP_CACHE: OpId = 13;
|
||||
pub const OP_FETCH_SOURCE_FILE: OpId = 14;
|
||||
pub const OP_FETCH_SOURCE_FILES: OpId = 14;
|
||||
pub const OP_OPEN: OpId = 15;
|
||||
pub const OP_CLOSE: OpId = 16;
|
||||
pub const OP_SEEK: OpId = 17;
|
||||
|
@ -133,8 +133,8 @@ pub fn dispatch(
|
|||
OP_CACHE => {
|
||||
dispatch_json::dispatch(compiler::op_cache, state, control, zero_copy)
|
||||
}
|
||||
OP_FETCH_SOURCE_FILE => dispatch_json::dispatch(
|
||||
compiler::op_fetch_source_file,
|
||||
OP_FETCH_SOURCE_FILES => dispatch_json::dispatch(
|
||||
compiler::op_fetch_source_files,
|
||||
state,
|
||||
control,
|
||||
zero_copy,
|
||||
|
|
139
js/compiler.ts
139
js/compiler.ts
|
@ -136,18 +136,25 @@ function fetchAsset(name: string): string {
|
|||
return sendSync(dispatch.OP_FETCH_ASSET, { name });
|
||||
}
|
||||
|
||||
/** Ops to Rust to resolve and fetch a modules meta data. */
|
||||
function fetchSourceFile(specifier: string, referrer: string): SourceFile {
|
||||
util.log("compiler.fetchSourceFile", { specifier, referrer });
|
||||
const res = sendSync(dispatch.OP_FETCH_SOURCE_FILE, {
|
||||
specifier,
|
||||
/** Ops to Rust to resolve and fetch modules meta data. */
|
||||
function fetchSourceFiles(
|
||||
specifiers: string[],
|
||||
referrer: string
|
||||
): SourceFile[] {
|
||||
util.log("compiler.fetchSourceFiles", { specifiers, referrer });
|
||||
const res = sendSync(dispatch.OP_FETCH_SOURCE_FILES, {
|
||||
specifiers,
|
||||
referrer
|
||||
});
|
||||
|
||||
return {
|
||||
...res,
|
||||
typeDirectives: parseTypeDirectives(res.sourceCode)
|
||||
};
|
||||
return res.map(
|
||||
(sourceFile: SourceFile): SourceFile => {
|
||||
return {
|
||||
...sourceFile,
|
||||
typeDirectives: parseTypeDirectives(sourceFile.sourceCode)
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/** Utility function to turn the number of bytes into a human readable
|
||||
|
@ -219,36 +226,71 @@ class Host implements ts.CompilerHost {
|
|||
|
||||
private _sourceFileCache: Record<string, SourceFile> = {};
|
||||
|
||||
private _resolveModule(specifier: string, referrer: string): SourceFile {
|
||||
util.log("host._resolveModule", { specifier, referrer });
|
||||
// Handle built-in assets specially.
|
||||
if (specifier.startsWith(ASSETS)) {
|
||||
const moduleName = specifier.split("/").pop()!;
|
||||
if (moduleName in this._sourceFileCache) {
|
||||
return this._sourceFileCache[moduleName];
|
||||
}
|
||||
const assetName = moduleName.includes(".")
|
||||
? moduleName
|
||||
: `${moduleName}.d.ts`;
|
||||
const sourceCode = fetchAsset(assetName);
|
||||
const sourceFile = {
|
||||
moduleName,
|
||||
filename: specifier,
|
||||
mediaType: MediaType.TypeScript,
|
||||
sourceCode
|
||||
};
|
||||
this._sourceFileCache[moduleName] = sourceFile;
|
||||
return sourceFile;
|
||||
}
|
||||
const sourceFile = fetchSourceFile(specifier, referrer);
|
||||
assert(sourceFile.moduleName != null);
|
||||
const { moduleName } = sourceFile;
|
||||
if (!(moduleName! in this._sourceFileCache)) {
|
||||
this._sourceFileCache[moduleName!] = sourceFile;
|
||||
private _getAsset(specifier: string): SourceFile {
|
||||
const moduleName = specifier.split("/").pop()!;
|
||||
if (moduleName in this._sourceFileCache) {
|
||||
return this._sourceFileCache[moduleName];
|
||||
}
|
||||
const assetName = moduleName.includes(".")
|
||||
? moduleName
|
||||
: `${moduleName}.d.ts`;
|
||||
const sourceCode = fetchAsset(assetName);
|
||||
const sourceFile = {
|
||||
moduleName,
|
||||
filename: specifier,
|
||||
mediaType: MediaType.TypeScript,
|
||||
sourceCode
|
||||
};
|
||||
this._sourceFileCache[moduleName] = sourceFile;
|
||||
return sourceFile;
|
||||
}
|
||||
|
||||
private _resolveModule(specifier: string, referrer: string): SourceFile {
|
||||
return this._resolveModules([specifier], referrer)[0];
|
||||
}
|
||||
|
||||
private _resolveModules(
|
||||
specifiers: string[],
|
||||
referrer: string
|
||||
): SourceFile[] {
|
||||
util.log("host._resolveModules", { specifiers, referrer });
|
||||
const resolvedModules: Array<SourceFile | undefined> = [];
|
||||
const modulesToRequest = [];
|
||||
|
||||
for (const specifier of specifiers) {
|
||||
// Firstly built-in assets are handled specially, so they should
|
||||
// be removed from array of files that we'll be requesting from Rust.
|
||||
if (specifier.startsWith(ASSETS)) {
|
||||
const assetFile = this._getAsset(specifier);
|
||||
resolvedModules.push(assetFile);
|
||||
} else if (specifier in this._sourceFileCache) {
|
||||
const module = this._sourceFileCache[specifier];
|
||||
resolvedModules.push(module);
|
||||
} else {
|
||||
// Temporarily fill with undefined, after fetching file from
|
||||
// Rust it will be filled with proper value.
|
||||
resolvedModules.push(undefined);
|
||||
modulesToRequest.push(specifier);
|
||||
}
|
||||
}
|
||||
|
||||
// Now get files from Rust.
|
||||
const sourceFiles = fetchSourceFiles(modulesToRequest, referrer);
|
||||
|
||||
for (const sourceFile of sourceFiles) {
|
||||
assert(sourceFile.moduleName != null);
|
||||
const { moduleName } = sourceFile;
|
||||
if (!(moduleName! in this._sourceFileCache)) {
|
||||
this._sourceFileCache[moduleName!] = sourceFile;
|
||||
}
|
||||
// And fill temporary `undefined`s with actual files.
|
||||
const index = resolvedModules.indexOf(undefined);
|
||||
resolvedModules[index] = sourceFile;
|
||||
}
|
||||
|
||||
return resolvedModules as SourceFile[];
|
||||
}
|
||||
|
||||
/* Deno specific APIs */
|
||||
|
||||
/** Provides the `ts.HostCompiler` interface for Deno.
|
||||
|
@ -371,22 +413,25 @@ class Host implements ts.CompilerHost {
|
|||
containingFile in this._sourceFileCache
|
||||
? this._sourceFileCache[containingFile].typeDirectives
|
||||
: undefined;
|
||||
return moduleNames.map(
|
||||
(moduleName): ts.ResolvedModuleFull | undefined => {
|
||||
const mappedModuleName = getMappedModuleName(
|
||||
moduleName,
|
||||
containingFile,
|
||||
typeDirectives
|
||||
);
|
||||
const sourceFile = this._resolveModule(
|
||||
mappedModuleName,
|
||||
containingFile
|
||||
);
|
||||
|
||||
const mappedModuleNames = moduleNames.map(
|
||||
(moduleName: string): string => {
|
||||
return getMappedModuleName(moduleName, containingFile, typeDirectives);
|
||||
}
|
||||
);
|
||||
|
||||
return this._resolveModules(mappedModuleNames, containingFile).map(
|
||||
(
|
||||
sourceFile: SourceFile,
|
||||
index: number
|
||||
): ts.ResolvedModuleFull | undefined => {
|
||||
if (sourceFile.moduleName) {
|
||||
const resolvedFileName = sourceFile.moduleName;
|
||||
// This flags to the compiler to not go looking to transpile functional
|
||||
// code, anything that is in `/$asset$/` is just library code
|
||||
const isExternalLibraryImport = moduleName.startsWith(ASSETS);
|
||||
const isExternalLibraryImport = mappedModuleNames[index].startsWith(
|
||||
ASSETS
|
||||
);
|
||||
const extension = getExtension(
|
||||
resolvedFileName,
|
||||
sourceFile.mediaType
|
||||
|
|
|
@ -16,7 +16,7 @@ export const OP_START = 10;
|
|||
export const OP_APPLY_SOURCE_MAP = 11;
|
||||
export const OP_FORMAT_ERROR = 12;
|
||||
export const OP_CACHE = 13;
|
||||
export const OP_FETCH_SOURCE_FILE = 14;
|
||||
export const OP_FETCH_SOURCE_FILES = 14;
|
||||
export const OP_OPEN = 15;
|
||||
export const OP_CLOSE = 16;
|
||||
export const OP_SEEK = 17;
|
||||
|
|
|
@ -3,10 +3,10 @@
|
|||
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
|
||||
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at [WILDCARD]compiler.ts:[WILDCARD]
|
||||
at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at resolveModuleNames ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])
|
||||
|
|
|
@ -3,9 +3,10 @@
|
|||
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
|
||||
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at [WILDCARD]compiler.ts:[WILDCARD]
|
||||
at resolveModuleNamesWorker ([WILDCARD])
|
||||
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])
|
||||
|
|
|
@ -3,9 +3,10 @@
|
|||
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
|
||||
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at [WILDCARD]compiler.ts:[WILDCARD]
|
||||
at resolveModuleNamesWorker ([WILDCARD])
|
||||
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])
|
||||
|
|
|
@ -3,9 +3,10 @@
|
|||
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
|
||||
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at [WILDCARD]compiler.ts:[WILDCARD]
|
||||
at resolveModuleNamesWorker ([WILDCARD])
|
||||
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])
|
||||
|
|
|
@ -3,9 +3,10 @@
|
|||
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
|
||||
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
|
||||
at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD])
|
||||
at [WILDCARD]compiler.ts:[WILDCARD]
|
||||
at resolveModuleNamesWorker ([WILDCARD])
|
||||
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])
|
||||
at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])
|
||||
|
|
Loading…
Reference in a new issue