development/deno
development/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno synced 2024-08-27 03:50:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tsc): do not store some typescript declaration file text in multiple places (#17410)

Browse source
This commit is contained in:
David Sherret 2023-01-14 09:36:19 -05:00 committed by GitHub
parent 429ccff657
commit 1712a88e69
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 173 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
cli/build.rs
View file

@ -94,6 +94,7 @@ mod ts {
"es2018.regexp",
"es2019.array",
"es2019",
"es2019.intl",
"es2019.object",
"es2019.string",
"es2019.symbol",
@ -116,6 +117,7 @@ mod ts {
"es2022.error",
"es2022.intl",
"es2022.object",
"es2022.sharedmemory",
"es2022.string",
"esnext",
"esnext.array",
@ -150,6 +152,15 @@ mod ts {
build_libs.push(op_lib.to_owned());
}
// used in the tests to verify that after snapshotting it has the same number
// of lib files loaded and hasn't included any ones lazily loaded from Rust
std::fs::write(
PathBuf::from(env::var_os("OUT_DIR").unwrap())
.join("lib_file_names.json"),
serde_json::to_string(&build_libs).unwrap(),
)
.unwrap();
#[op]
fn op_build_info(state: &mut OpState) -> Value {
let build_specifier = "asset:///bootstrap.ts";
@ -196,7 +207,7 @@ mod ts {
// we need a basic file to send to tsc to warm it up.
if args.specifier == build_specifier {
Ok(json!({
"data": r#"console.log("hello deno!");"#,
"data": r#"Deno.writeTextFile("hello.txt", "hello deno!");"#,
"version": "1",
// this corresponds to `ts.ScriptKind.TypeScript`
"scriptKind": 3
@ -423,51 +434,6 @@ fn main() {
println!("cargo:rustc-env=TS_VERSION={}", ts::version());
println!("cargo:rerun-if-env-changed=TS_VERSION");
println!(
"cargo:rustc-env=DENO_CONSOLE_LIB_PATH={}",
deno_console::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_URL_LIB_PATH={}",
deno_url::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEB_LIB_PATH={}",
deno_web::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_FETCH_LIB_PATH={}",
deno_fetch::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBGPU_LIB_PATH={}",
deno_webgpu_get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBSOCKET_LIB_PATH={}",
deno_websocket::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBSTORAGE_LIB_PATH={}",
deno_webstorage::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_CACHE_LIB_PATH={}",
deno_cache::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_CRYPTO_LIB_PATH={}",
deno_crypto::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_BROADCAST_CHANNEL_LIB_PATH={}",
deno_broadcast_channel::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_NET_LIB_PATH={}",
deno_net::get_declaration().display()
);
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());

29
cli/lsp/tsc.rs
View file

@ -208,7 +208,7 @@ impl AssetDocument {
type AssetsMap = HashMap<ModuleSpecifier, AssetDocument>;
fn new_assets_map() -> Arc<Mutex<AssetsMap>> {
let assets = tsc::STATIC_ASSETS
let assets = tsc::LAZILY_LOADED_STATIC_ASSETS
.iter()
.map(|(k, v)| {
let url_str = format!("asset:///{}", k);
@ -3455,6 +3455,8 @@ mod tests {
use crate::lsp::documents::Documents;
use crate::lsp::documents::LanguageId;
use crate::lsp::text::LineIndex;
use crate::tsc::AssetText;
use pretty_assertions::assert_eq;
use std::path::Path;
use std::path::PathBuf;
use test_util::TempDir;
@ -3913,18 +3915,31 @@ mod tests {
Default::default(),
)
.unwrap();
let assets = result.as_array().unwrap();
let assets: Vec<AssetText> = serde_json::from_value(result).unwrap();
let mut asset_names = assets
.iter()
.map(|a| {
a.specifier
.replace("asset:///lib.", "")
.replace(".d.ts", "")
})
.collect::<Vec<_>>();
let mut expected_asset_names: Vec<String> = serde_json::from_str(
include_str!(concat!(env!("OUT_DIR"), "/lib_file_names.json")),
)
.unwrap();
asset_names.sort();
expected_asset_names.sort();
// You might have found this assertion starts failing after upgrading TypeScript.
// Just update the new number of assets (declaration files) for this number.
assert_eq!(assets.len(), 72);
// Ensure build.rs is updated so these match.
assert_eq!(asset_names, expected_asset_names);
// get some notification when the size of the assets grows
let mut total_size = 0;
for asset in assets {
let obj = asset.as_object().unwrap();
let text = obj.get("text").unwrap().as_str().unwrap();
total_size += text.len();
total_size += asset.text.len();
}
assert!(total_size > 0);
assert!(total_size < 2_000_000); // currently as of TS 4.6, it's 0.7MB

43
cli/tsc/99_main_compiler.js
View file

@ -385,7 +385,7 @@ delete Object.prototype.__proto__;
// paths must be either relative or absolute. Since
// analysis in Rust operates on fully resolved URLs,
// it makes sense to use the same scheme here.
const ASSETS = "asset:///";
const ASSETS_URL_PREFIX = "asset:///";
/** Diagnostics that are intentionally ignored when compiling TypeScript in
* Deno, as they provide misleading or incorrect information. */
@ -431,6 +431,7 @@ delete Object.prototype.__proto__;
noEmit: true,
strict: true,
target: ts.ScriptTarget.ESNext,
lib: ["lib.deno.window.d.ts"],
};
// todo(dsherret): can we remove this and just use ts.OperationCanceledException?
@ -546,10 +547,10 @@ delete Object.prototype.__proto__;
return sourceFile;
},
getDefaultLibFileName() {
return `${ASSETS}/lib.esnext.d.ts`;
return `${ASSETS_URL_PREFIX}lib.esnext.d.ts`;
},
getDefaultLibLocation() {
return ASSETS;
return ASSETS_URL_PREFIX;
},
writeFile(fileName, data, _writeByteOrderMark, _onError, _sourceFiles) {
if (logDebug) {
@ -887,6 +888,20 @@ delete Object.prototype.__proto__;
debug("<<< exec stop");
}
function getAssets() {
/** @type {{ specifier: string; text: string; }[]} */
const assets = [];
for (const sourceFile of sourceFileCache.values()) {
if (sourceFile.fileName.startsWith(ASSETS_URL_PREFIX)) {
assets.push({
specifier: sourceFile.fileName,
text: sourceFile.text,
});
}
}
return assets;
}
/**
* @param {number} id
* @param {any} data
@ -935,16 +950,7 @@ delete Object.prototype.__proto__;
);
}
case "getAssets": {
const assets = [];
for (const sourceFile of sourceFileCache.values()) {
if (sourceFile.fileName.startsWith(ASSETS)) {
assets.push({
specifier: sourceFile.fileName,
text: sourceFile.text,
});
}
}
return respond(id, assets);
return respond(id, getAssets());
}
case "getApplicableRefactors": {
return respond(
@ -1281,7 +1287,10 @@ delete Object.prototype.__proto__;
// we are caching in memory common type libraries that will be re-used by
// tsc on when the snapshot is restored
assert(
host.getSourceFile(`${ASSETS}${specifier}`, ts.ScriptTarget.ESNext),
host.getSourceFile(
`${ASSETS_URL_PREFIX}${specifier}`,
ts.ScriptTarget.ESNext,
),
);
}
// this helps ensure as much as possible is in memory that is re-usable
@ -1292,12 +1301,16 @@ delete Object.prototype.__proto__;
options: SNAPSHOT_COMPILE_OPTIONS,
host,
});
ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM);
assert(ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM).length === 0);
// remove this now that we don't need it anymore for warming up tsc
sourceFileCache.delete(buildSpecifier);
// exposes the two functions that are called by `tsc::exec()` when type
// checking TypeScript.
globalThis.startup = startup;
globalThis.exec = exec;
globalThis.getAssets = getAssets;
// exposes the functions that are called when the compiler is used as a
// language service.

200
cli/tsc/mod.rs
View file

@ -25,6 +25,7 @@ use deno_core::serde::Serializer;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::serde_v8;
use deno_core::Extension;
use deno_core::JsRuntime;
use deno_core::ModuleSpecifier;
@ -49,28 +50,6 @@ pub use self::diagnostics::DiagnosticMessageChain;
pub use self::diagnostics::Diagnostics;
pub use self::diagnostics::Position;
// Declaration files
pub static DENO_NS_LIB: &str = include_str!("dts/lib.deno.ns.d.ts");
pub static DENO_CONSOLE_LIB: &str = include_str!(env!("DENO_CONSOLE_LIB_PATH"));
pub static DENO_URL_LIB: &str = include_str!(env!("DENO_URL_LIB_PATH"));
pub static DENO_WEB_LIB: &str = include_str!(env!("DENO_WEB_LIB_PATH"));
pub static DENO_FETCH_LIB: &str = include_str!(env!("DENO_FETCH_LIB_PATH"));
pub static DENO_WEBGPU_LIB: &str = include_str!(env!("DENO_WEBGPU_LIB_PATH"));
pub static DENO_WEBSOCKET_LIB: &str =
include_str!(env!("DENO_WEBSOCKET_LIB_PATH"));
pub static DENO_WEBSTORAGE_LIB: &str =
include_str!(env!("DENO_WEBSTORAGE_LIB_PATH"));
pub static DENO_CACHE_LIB: &str = include_str!(env!("DENO_CACHE_LIB_PATH"));
pub static DENO_CRYPTO_LIB: &str = include_str!(env!("DENO_CRYPTO_LIB_PATH"));
pub static DENO_BROADCAST_CHANNEL_LIB: &str =
include_str!(env!("DENO_BROADCAST_CHANNEL_LIB_PATH"));
pub static DENO_NET_LIB: &str = include_str!(env!("DENO_NET_LIB_PATH"));
pub static SHARED_GLOBALS_LIB: &str =
include_str!("dts/lib.deno.shared_globals.d.ts");
pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
#[cold]
#[inline(never)]
@ -89,28 +68,57 @@ pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
);
pub fn get_types_declaration_file_text(unstable: bool) -> String {
let mut types = vec![
DENO_NS_LIB,
DENO_CONSOLE_LIB,
DENO_URL_LIB,
DENO_WEB_LIB,
DENO_FETCH_LIB,
DENO_WEBGPU_LIB,
DENO_WEBSOCKET_LIB,
DENO_WEBSTORAGE_LIB,
DENO_CRYPTO_LIB,
DENO_BROADCAST_CHANNEL_LIB,
DENO_NET_LIB,
SHARED_GLOBALS_LIB,
DENO_CACHE_LIB,
WINDOW_LIB,
let mut assets = get_asset_texts_from_new_runtime()
.unwrap()
.into_iter()
.map(|a| (a.specifier, a.text))
.collect::<HashMap<_, _>>();
let mut lib_names = vec![
"deno.ns",
"deno.console",
"deno.url",
"deno.web",
"deno.fetch",
"deno.webgpu",
"deno.websocket",
"deno.webstorage",
"deno.crypto",
"deno.broadcast_channel",
"deno.net",
"deno.shared_globals",
"deno.cache",
"deno.window",
];
if unstable {
types.push(UNSTABLE_NS_LIB);
lib_names.push("deno.unstable");
}
types.join("\n")
lib_names
.into_iter()
.map(|name| {
let asset_url = format!("asset:///lib.{}.d.ts", name);
assets.remove(&asset_url).unwrap()
})
.collect::<Vec<_>>()
.join("\n")
}
fn get_asset_texts_from_new_runtime() -> Result<Vec<AssetText>, AnyError> {
// the assets are stored within the typescript isolate, so take them out of there
let mut runtime = JsRuntime::new(RuntimeOptions {
startup_snapshot: Some(compiler_snapshot()),
extensions: vec![Extension::builder("deno_cli_tsc")
.ops(get_tsc_ops())
.build()],
..Default::default()
});
let global =
runtime.execute_script("get_assets.js", "globalThis.getAssets()")?;
let scope = &mut runtime.handle_scope();
let local = deno_core::v8::Local::new(scope, global);
Ok(serde_v8::from_v8::<Vec<AssetText>>(scope, local)?)
}
pub fn compiler_snapshot() -> Snapshot {
@ -124,40 +132,44 @@ macro_rules! inc {
}
/// Contains static assets that are not preloaded in the compiler snapshot.
pub static STATIC_ASSETS: Lazy<HashMap<&'static str, &'static str>> =
Lazy::new(|| {
([
(
"lib.dom.asynciterable.d.ts",
inc!("lib.dom.asynciterable.d.ts"),
),
("lib.dom.d.ts", inc!("lib.dom.d.ts")),
("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")),
("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")),
("lib.es6.d.ts", inc!("lib.es6.d.ts")),
("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")),
("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")),
("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")),
("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")),
("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")),
("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")),
("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")),
("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")),
("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")),
("lib.webworker.d.ts", inc!("lib.webworker.d.ts")),
(
"lib.webworker.importscripts.d.ts",
inc!("lib.webworker.importscripts.d.ts"),
),
(
"lib.webworker.iterable.d.ts",
inc!("lib.webworker.iterable.d.ts"),
),
])
.iter()
.cloned()
.collect()
});
///
/// We lazily load these because putting them in the compiler snapshot will
/// increase memory usage when not used (last time checked by about 0.5MB).
pub static LAZILY_LOADED_STATIC_ASSETS: Lazy<
HashMap<&'static str, &'static str>,
> = Lazy::new(|| {
([
(
"lib.dom.asynciterable.d.ts",
inc!("lib.dom.asynciterable.d.ts"),
),
("lib.dom.d.ts", inc!("lib.dom.d.ts")),
("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")),
("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")),
("lib.es6.d.ts", inc!("lib.es6.d.ts")),
("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")),
("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")),
("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")),
("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")),
("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")),
("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")),
("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")),
("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")),
("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")),
("lib.webworker.d.ts", inc!("lib.webworker.d.ts")),
(
"lib.webworker.importscripts.d.ts",
inc!("lib.webworker.importscripts.d.ts"),
),
(
"lib.webworker.iterable.d.ts",
inc!("lib.webworker.iterable.d.ts"),
),
])
.iter()
.cloned()
.collect()
});
/// A structure representing stats from a type check operation for a graph.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
@ -193,9 +205,16 @@ impl fmt::Display for Stats {
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssetText {
pub specifier: String,
pub text: String,
}
/// Retrieve a static asset that are included in the binary.
pub fn get_asset(asset: &str) -> Option<&'static str> {
STATIC_ASSETS.get(asset).map(|s| s.to_owned())
fn get_lazily_loaded_asset(asset: &str) -> Option<&'static str> {
LAZILY_LOADED_STATIC_ASSETS.get(asset).map(|s| s.to_owned())
}
fn get_maybe_hash(
@ -501,9 +520,8 @@ fn op_load(state: &mut OpState, args: Value) -> Result<Value, AnyError> {
hash = Some("1".to_string());
media_type = MediaType::Dts;
Some(Cow::Borrowed("declare const __: any;\nexport = __;\n"))
} else if v.specifier.starts_with("asset:///") {
let name = v.specifier.replace("asset:///", "");
let maybe_source = get_asset(&name);
} else if let Some(name) = v.specifier.strip_prefix("asset:///") {
let maybe_source = get_lazily_loaded_asset(name);
hash = get_maybe_hash(maybe_source, &state.hash_data);
media_type = MediaType::from(&v.specifier);
maybe_source.map(Cow::Borrowed)
@ -774,16 +792,7 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
let mut runtime = JsRuntime::new(RuntimeOptions {
startup_snapshot: Some(compiler_snapshot()),
extensions: vec![Extension::builder("deno_cli_tsc")
.ops(vec![
op_cwd::decl(),
op_create_hash::decl(),
op_emit::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_resolve::decl(),
op_respond::decl(),
])
.ops(get_tsc_ops())
.state(move |state| {
state.put(State::new(
request.graph_data.clone(),
@ -833,6 +842,19 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
}
}
fn get_tsc_ops() -> Vec<deno_core::OpDecl> {
vec![
op_cwd::decl(),
op_create_hash::decl(),
op_emit::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_resolve::decl(),
op_respond::decl(),
]
}
#[cfg(test)]
mod tests {
use super::Diagnostic;
@ -1089,7 +1111,7 @@ mod tests {
.expect("should have invoked op");
let actual: LoadResponse =
serde_json::from_value(value).expect("failed to deserialize");
let expected = get_asset("lib.dom.d.ts").unwrap();
let expected = get_lazily_loaded_asset("lib.dom.d.ts").unwrap();
assert_eq!(actual.data, expected);
assert!(actual.version.is_some());
assert_eq!(actual.script_kind, 3);