mirror of
https://github.com/denoland/deno
synced 2024-08-24 18:47:06 +00:00
refactor(snapshots): to their own crate (#14794)
Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>
This commit is contained in:
parent
d39094913e
commit
fd5a12d7e2
24
Cargo.lock
generated
24
Cargo.lock
generated
|
@ -738,23 +738,14 @@ dependencies = [
|
|||
"data-url",
|
||||
"deno_ast",
|
||||
"deno_bench_util",
|
||||
"deno_broadcast_channel",
|
||||
"deno_console",
|
||||
"deno_core",
|
||||
"deno_crypto",
|
||||
"deno_doc",
|
||||
"deno_emit",
|
||||
"deno_fetch",
|
||||
"deno_graph",
|
||||
"deno_lint",
|
||||
"deno_net",
|
||||
"deno_runtime",
|
||||
"deno_snapshots",
|
||||
"deno_task_shell",
|
||||
"deno_url",
|
||||
"deno_web",
|
||||
"deno_webgpu",
|
||||
"deno_websocket",
|
||||
"deno_webstorage",
|
||||
"dissimilar",
|
||||
"dprint-plugin-json",
|
||||
"dprint-plugin-markdown",
|
||||
|
@ -1116,6 +1107,19 @@ dependencies = [
|
|||
"winres",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deno_snapshots"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"deno_core",
|
||||
"deno_runtime",
|
||||
"lzzzz",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deno_task_shell"
|
||||
version = "0.4.0"
|
||||
|
|
|
@ -9,6 +9,7 @@ members = [
|
|||
"ops",
|
||||
"runtime",
|
||||
"serde_v8",
|
||||
"snapshots",
|
||||
"test_ffi",
|
||||
"test_util",
|
||||
"ext/broadcast_channel",
|
||||
|
|
|
@ -25,17 +25,7 @@ harness = false
|
|||
path = "./bench/lsp_bench_standalone.rs"
|
||||
|
||||
[build-dependencies]
|
||||
deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" }
|
||||
deno_console = { version = "0.58.0", path = "../ext/console" }
|
||||
deno_core = { version = "0.140.0", path = "../core" }
|
||||
deno_crypto = { version = "0.72.0", path = "../ext/crypto" }
|
||||
deno_fetch = { version = "0.81.0", path = "../ext/fetch" }
|
||||
deno_net = { version = "0.50.0", path = "../ext/net" }
|
||||
deno_url = { version = "0.58.0", path = "../ext/url" }
|
||||
deno_web = { version = "0.89.0", path = "../ext/web" }
|
||||
deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" }
|
||||
deno_websocket = { version = "0.63.0", path = "../ext/websocket" }
|
||||
deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" }
|
||||
deno_runtime = { version = "0.66.0", path = "../runtime" }
|
||||
regex = "=1.5.6"
|
||||
serde = { version = "=1.0.136", features = ["derive"] }
|
||||
zstd = '=0.11.1'
|
||||
|
@ -52,6 +42,7 @@ deno_emit = "0.3.0"
|
|||
deno_graph = "0.28.0"
|
||||
deno_lint = { version = "0.31.0", features = ["docs"] }
|
||||
deno_runtime = { version = "0.66.0", path = "../runtime" }
|
||||
deno_snapshots = { version = "0.0.0", path = "../snapshots" }
|
||||
deno_task_shell = "0.4.0"
|
||||
|
||||
atty = "=0.2.14"
|
||||
|
|
387
cli/build.rs
387
cli/build.rs
|
@ -1,323 +1,19 @@
|
|||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
use deno_core::error::custom_error;
|
||||
use deno_core::error::AnyError;
|
||||
use deno_core::op;
|
||||
use deno_core::serde::Deserialize;
|
||||
use deno_core::serde_json::json;
|
||||
use deno_core::serde_json::Value;
|
||||
use deno_core::Extension;
|
||||
use deno_core::JsRuntime;
|
||||
use deno_core::OpState;
|
||||
use deno_core::RuntimeOptions;
|
||||
use regex::Regex;
|
||||
use std::collections::HashMap;
|
||||
use deno_runtime::deno_broadcast_channel;
|
||||
use deno_runtime::deno_console;
|
||||
use deno_runtime::deno_crypto;
|
||||
use deno_runtime::deno_fetch;
|
||||
use deno_runtime::deno_net;
|
||||
use deno_runtime::deno_url;
|
||||
use deno_runtime::deno_web;
|
||||
use deno_runtime::deno_websocket;
|
||||
use deno_runtime::deno_webstorage;
|
||||
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
// TODO(bartlomieju): this module contains a lot of duplicated
|
||||
// logic with `runtime/build.rs`, factor out to `deno_core`.
|
||||
fn create_snapshot(
|
||||
mut js_runtime: JsRuntime,
|
||||
snapshot_path: &Path,
|
||||
files: Vec<PathBuf>,
|
||||
) {
|
||||
// TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the
|
||||
// workspace root.
|
||||
let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
|
||||
for file in files {
|
||||
println!("cargo:rerun-if-changed={}", file.display());
|
||||
let display_path = file.strip_prefix(display_root).unwrap();
|
||||
let display_path_str = display_path.display().to_string();
|
||||
js_runtime
|
||||
.execute_script(
|
||||
&("deno:".to_string() + &display_path_str.replace('\\', "/")),
|
||||
&std::fs::read_to_string(&file).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let snapshot = js_runtime.snapshot();
|
||||
let snapshot_slice: &[u8] = &*snapshot;
|
||||
println!("Snapshot size: {}", snapshot_slice.len());
|
||||
|
||||
let compressed_snapshot_with_size = {
|
||||
let mut vec = vec![];
|
||||
|
||||
vec.extend_from_slice(
|
||||
&u32::try_from(snapshot.len())
|
||||
.expect("snapshot larger than 4gb")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
|
||||
vec.extend_from_slice(
|
||||
&zstd::bulk::compress(snapshot_slice, 22)
|
||||
.expect("snapshot compression failed"),
|
||||
);
|
||||
|
||||
vec
|
||||
};
|
||||
|
||||
println!(
|
||||
"Snapshot compressed size: {}",
|
||||
compressed_snapshot_with_size.len()
|
||||
);
|
||||
|
||||
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
|
||||
println!("Snapshot written to: {} ", snapshot_path.display());
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LoadArgs {
|
||||
/// The fully qualified specifier that should be loaded.
|
||||
specifier: String,
|
||||
}
|
||||
|
||||
fn create_compiler_snapshot(
|
||||
snapshot_path: &Path,
|
||||
files: Vec<PathBuf>,
|
||||
cwd: &Path,
|
||||
) {
|
||||
// libs that are being provided by op crates.
|
||||
let mut op_crate_libs = HashMap::new();
|
||||
op_crate_libs.insert("deno.console", deno_console::get_declaration());
|
||||
op_crate_libs.insert("deno.url", deno_url::get_declaration());
|
||||
op_crate_libs.insert("deno.web", deno_web::get_declaration());
|
||||
op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
|
||||
op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration());
|
||||
op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration());
|
||||
op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration());
|
||||
op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration());
|
||||
op_crate_libs.insert(
|
||||
"deno.broadcast_channel",
|
||||
deno_broadcast_channel::get_declaration(),
|
||||
);
|
||||
op_crate_libs.insert("deno.net", deno_net::get_declaration());
|
||||
|
||||
// ensure we invalidate the build properly.
|
||||
for (_, path) in op_crate_libs.iter() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
|
||||
// libs that should be loaded into the isolate before snapshotting.
|
||||
let libs = vec![
|
||||
// Deno custom type libraries
|
||||
"deno.window",
|
||||
"deno.worker",
|
||||
"deno.shared_globals",
|
||||
"deno.ns",
|
||||
"deno.unstable",
|
||||
// Deno built-in type libraries
|
||||
"es5",
|
||||
"es2015.collection",
|
||||
"es2015.core",
|
||||
"es2015",
|
||||
"es2015.generator",
|
||||
"es2015.iterable",
|
||||
"es2015.promise",
|
||||
"es2015.proxy",
|
||||
"es2015.reflect",
|
||||
"es2015.symbol",
|
||||
"es2015.symbol.wellknown",
|
||||
"es2016.array.include",
|
||||
"es2016",
|
||||
"es2017",
|
||||
"es2017.intl",
|
||||
"es2017.object",
|
||||
"es2017.sharedmemory",
|
||||
"es2017.string",
|
||||
"es2017.typedarrays",
|
||||
"es2018.asyncgenerator",
|
||||
"es2018.asynciterable",
|
||||
"es2018",
|
||||
"es2018.intl",
|
||||
"es2018.promise",
|
||||
"es2018.regexp",
|
||||
"es2019.array",
|
||||
"es2019",
|
||||
"es2019.object",
|
||||
"es2019.string",
|
||||
"es2019.symbol",
|
||||
"es2020.bigint",
|
||||
"es2020",
|
||||
"es2020.date",
|
||||
"es2020.intl",
|
||||
"es2020.number",
|
||||
"es2020.promise",
|
||||
"es2020.sharedmemory",
|
||||
"es2020.string",
|
||||
"es2020.symbol.wellknown",
|
||||
"es2021",
|
||||
"es2021.intl",
|
||||
"es2021.promise",
|
||||
"es2021.string",
|
||||
"es2021.weakref",
|
||||
"es2022",
|
||||
"es2022.array",
|
||||
"es2022.error",
|
||||
"es2022.intl",
|
||||
"es2022.object",
|
||||
"es2022.string",
|
||||
"esnext",
|
||||
"esnext.array",
|
||||
"esnext.intl",
|
||||
];
|
||||
|
||||
let path_dts = cwd.join("dts");
|
||||
// ensure we invalidate the build properly.
|
||||
for name in libs.iter() {
|
||||
println!(
|
||||
"cargo:rerun-if-changed={}",
|
||||
path_dts.join(format!("lib.{}.d.ts", name)).display()
|
||||
);
|
||||
}
|
||||
|
||||
// create a copy of the vector that includes any op crate libs to be passed
|
||||
// to the JavaScript compiler to build into the snapshot
|
||||
let mut build_libs = libs.clone();
|
||||
for (op_lib, _) in op_crate_libs.iter() {
|
||||
build_libs.push(op_lib.to_owned());
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_build_info(state: &mut OpState) -> Value {
|
||||
let build_specifier = "asset:///bootstrap.ts";
|
||||
let build_libs = state.borrow::<Vec<&str>>();
|
||||
json!({
|
||||
"buildSpecifier": build_specifier,
|
||||
"libs": build_libs,
|
||||
})
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_cwd() -> String {
|
||||
"cache:///".into()
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_exists() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_script_version(
|
||||
_state: &mut OpState,
|
||||
_args: Value,
|
||||
) -> Result<Option<String>, AnyError> {
|
||||
Ok(Some("1".to_string()))
|
||||
}
|
||||
|
||||
#[op]
|
||||
// using the same op that is used in `tsc.rs` for loading modules and reading
|
||||
// files, but a slightly different implementation at build time.
|
||||
fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> {
|
||||
let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>();
|
||||
let path_dts = state.borrow::<PathBuf>();
|
||||
let re_asset =
|
||||
Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
|
||||
let build_specifier = "asset:///bootstrap.ts";
|
||||
|
||||
// we need a basic file to send to tsc to warm it up.
|
||||
if args.specifier == build_specifier {
|
||||
Ok(json!({
|
||||
"data": r#"console.log("hello deno!");"#,
|
||||
"version": "1",
|
||||
// this corresponds to `ts.ScriptKind.TypeScript`
|
||||
"scriptKind": 3
|
||||
}))
|
||||
// specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to
|
||||
// parse out just the name so we can lookup the asset.
|
||||
} else if let Some(caps) = re_asset.captures(&args.specifier) {
|
||||
if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
|
||||
// if it comes from an op crate, we were supplied with the path to the
|
||||
// file.
|
||||
let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
|
||||
PathBuf::from(op_crate_lib).canonicalize().unwrap()
|
||||
// otherwise we are will generate the path ourself
|
||||
} else {
|
||||
path_dts.join(format!("lib.{}.d.ts", lib))
|
||||
};
|
||||
let data = std::fs::read_to_string(path)?;
|
||||
Ok(json!({
|
||||
"data": data,
|
||||
"version": "1",
|
||||
// this corresponds to `ts.ScriptKind.TypeScript`
|
||||
"scriptKind": 3
|
||||
}))
|
||||
} else {
|
||||
Err(custom_error(
|
||||
"InvalidSpecifier",
|
||||
format!("An invalid specifier was requested: {}", args.specifier),
|
||||
))
|
||||
}
|
||||
} else {
|
||||
Err(custom_error(
|
||||
"InvalidSpecifier",
|
||||
format!("An invalid specifier was requested: {}", args.specifier),
|
||||
))
|
||||
}
|
||||
}
|
||||
let js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
will_snapshot: true,
|
||||
extensions: vec![Extension::builder()
|
||||
.ops(vec![
|
||||
op_build_info::decl(),
|
||||
op_cwd::decl(),
|
||||
op_exists::decl(),
|
||||
op_load::decl(),
|
||||
op_script_version::decl(),
|
||||
])
|
||||
.state(move |state| {
|
||||
state.put(op_crate_libs.clone());
|
||||
state.put(build_libs.clone());
|
||||
state.put(path_dts.clone());
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.build()],
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
create_snapshot(js_runtime, snapshot_path, files);
|
||||
}
|
||||
|
||||
fn ts_version() -> String {
|
||||
std::fs::read_to_string("tsc/00_typescript.js")
|
||||
.unwrap()
|
||||
.lines()
|
||||
.find(|l| l.contains("ts.version = "))
|
||||
.expect(
|
||||
"Failed to find the pattern `ts.version = ` in typescript source code",
|
||||
)
|
||||
.chars()
|
||||
.skip_while(|c| !char::is_numeric(*c))
|
||||
.take_while(|c| *c != '"')
|
||||
.collect::<String>()
|
||||
}
|
||||
|
||||
fn git_commit_hash() -> String {
|
||||
if let Ok(output) = std::process::Command::new("git")
|
||||
.arg("rev-list")
|
||||
.arg("-1")
|
||||
.arg("HEAD")
|
||||
.output()
|
||||
{
|
||||
if output.status.success() {
|
||||
std::str::from_utf8(&output.stdout[..40])
|
||||
.unwrap()
|
||||
.to_string()
|
||||
} else {
|
||||
// When not in git repository
|
||||
// (e.g. when the user install by `cargo install deno`)
|
||||
"UNKNOWN".to_string()
|
||||
}
|
||||
} else {
|
||||
// When there is no git command for some reason
|
||||
"UNKNOWN".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// Skip building from docs.rs.
|
||||
if env::var_os("DOCS_RS").is_some() {
|
||||
|
@ -330,8 +26,9 @@ fn main() {
|
|||
if target != host {
|
||||
panic!("Cross compiling with snapshot is not supported.");
|
||||
}
|
||||
// To debug snapshot issues uncomment:
|
||||
// op_fetch_asset::trace_serializer();
|
||||
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
|
||||
if let Ok(c) = env::var("DENO_CANARY") {
|
||||
println!("cargo:rustc-env=DENO_CANARY={}", c);
|
||||
|
@ -385,18 +82,6 @@ fn main() {
|
|||
deno_net::get_declaration().display()
|
||||
);
|
||||
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
|
||||
let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
|
||||
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
|
||||
|
||||
// Main snapshot
|
||||
let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
|
||||
|
||||
let js_files = get_js_files("tsc");
|
||||
create_compiler_snapshot(&compiler_snapshot_path, js_files, &c);
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let mut res = winres::WindowsResource::new();
|
||||
|
@ -414,16 +99,38 @@ fn deno_webgpu_get_declaration() -> PathBuf {
|
|||
manifest_dir.join("dts").join("lib.deno_webgpu.d.ts")
|
||||
}
|
||||
|
||||
fn get_js_files(d: &str) -> Vec<PathBuf> {
|
||||
let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut js_files = std::fs::read_dir(d)
|
||||
.unwrap()
|
||||
.map(|dir_entry| {
|
||||
let file = dir_entry.unwrap();
|
||||
manifest_dir.join(file.path())
|
||||
})
|
||||
.filter(|path| path.extension().unwrap_or_default() == "js")
|
||||
.collect::<Vec<PathBuf>>();
|
||||
js_files.sort();
|
||||
js_files
|
||||
fn git_commit_hash() -> String {
|
||||
if let Ok(output) = std::process::Command::new("git")
|
||||
.arg("rev-list")
|
||||
.arg("-1")
|
||||
.arg("HEAD")
|
||||
.output()
|
||||
{
|
||||
if output.status.success() {
|
||||
std::str::from_utf8(&output.stdout[..40])
|
||||
.unwrap()
|
||||
.to_string()
|
||||
} else {
|
||||
// When not in git repository
|
||||
// (e.g. when the user install by `cargo install deno`)
|
||||
"UNKNOWN".to_string()
|
||||
}
|
||||
} else {
|
||||
// When there is no git command for some reason
|
||||
"UNKNOWN".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_version() -> String {
|
||||
std::fs::read_to_string("tsc/00_typescript.js")
|
||||
.unwrap()
|
||||
.lines()
|
||||
.find(|l| l.contains("ts.version = "))
|
||||
.expect(
|
||||
"Failed to find the pattern `ts.version = ` in typescript source code",
|
||||
)
|
||||
.chars()
|
||||
.skip_while(|c| !char::is_numeric(*c))
|
||||
.take_while(|c| *c != '"')
|
||||
.collect::<String>()
|
||||
}
|
||||
|
|
|
@ -2618,7 +2618,7 @@ fn op_script_version(
|
|||
fn js_runtime(performance: Arc<Performance>) -> JsRuntime {
|
||||
JsRuntime::new(RuntimeOptions {
|
||||
extensions: vec![init_extension(performance)],
|
||||
startup_snapshot: Some(tsc::compiler_snapshot()),
|
||||
startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
|
|
@ -179,6 +179,7 @@ fn create_web_worker_callback(
|
|||
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
|
||||
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
|
||||
stdio: stdio.clone(),
|
||||
startup_snapshot: Some(deno_snapshots::cli_snapshot()),
|
||||
};
|
||||
|
||||
WebWorker::bootstrap_from_options(
|
||||
|
@ -274,6 +275,7 @@ pub fn create_main_worker(
|
|||
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
|
||||
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
|
||||
stdio,
|
||||
startup_snapshot: Some(deno_snapshots::cli_snapshot()),
|
||||
};
|
||||
|
||||
MainWorker::bootstrap_from_options(main_module, permissions, options)
|
||||
|
|
|
@ -307,6 +307,7 @@ pub async fn run(
|
|||
shared_array_buffer_store: None,
|
||||
compiled_wasm_module_store: None,
|
||||
stdio: Default::default(),
|
||||
startup_snapshot: Some(deno_snapshots::cli_snapshot()),
|
||||
};
|
||||
let mut worker = MainWorker::bootstrap_from_options(
|
||||
main_module.clone(),
|
||||
|
|
28
cli/tsc.rs
28
cli/tsc.rs
|
@ -24,7 +24,6 @@ use deno_core::JsRuntime;
|
|||
use deno_core::ModuleSpecifier;
|
||||
use deno_core::OpState;
|
||||
use deno_core::RuntimeOptions;
|
||||
use deno_core::Snapshot;
|
||||
use deno_graph::Resolved;
|
||||
use once_cell::sync::Lazy;
|
||||
use std::collections::HashMap;
|
||||
|
@ -52,27 +51,6 @@ pub static SHARED_GLOBALS_LIB: &str =
|
|||
pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
|
||||
pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
|
||||
|
||||
pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
|| {
|
||||
static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
|
||||
|
||||
zstd::bulk::decompress(
|
||||
&COMPRESSED_COMPILER_SNAPSHOT[4..],
|
||||
u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
|
||||
as usize,
|
||||
)
|
||||
.unwrap()
|
||||
.into_boxed_slice()
|
||||
},
|
||||
);
|
||||
|
||||
pub fn compiler_snapshot() -> Snapshot {
|
||||
Snapshot::Static(&*COMPILER_SNAPSHOT)
|
||||
}
|
||||
|
||||
macro_rules! inc {
|
||||
($e:expr) => {
|
||||
include_str!(concat!("dts/", $e))
|
||||
|
@ -657,7 +635,7 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
|
|||
})
|
||||
.collect();
|
||||
let mut runtime = JsRuntime::new(RuntimeOptions {
|
||||
startup_snapshot: Some(compiler_snapshot()),
|
||||
startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
|
||||
extensions: vec![Extension::builder()
|
||||
.ops(vec![
|
||||
op_cwd::decl(),
|
||||
|
@ -841,9 +819,9 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn test_compiler_snapshot() {
|
||||
fn test_tsc_snapshot() {
|
||||
let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
|
||||
startup_snapshot: Some(compiler_snapshot()),
|
||||
startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
|
||||
..Default::default()
|
||||
});
|
||||
js_runtime
|
||||
|
|
|
@ -21,25 +21,6 @@ path = "lib.rs"
|
|||
name = "hello_runtime"
|
||||
path = "examples/hello_runtime.rs"
|
||||
|
||||
[build-dependencies]
|
||||
deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" }
|
||||
deno_console = { version = "0.58.0", path = "../ext/console" }
|
||||
deno_core = { version = "0.140.0", path = "../core" }
|
||||
deno_crypto = { version = "0.72.0", path = "../ext/crypto" }
|
||||
deno_fetch = { version = "0.81.0", path = "../ext/fetch" }
|
||||
deno_ffi = { version = "0.45.0", path = "../ext/ffi" }
|
||||
deno_http = { version = "0.52.0", path = "../ext/http" }
|
||||
deno_net = { version = "0.50.0", path = "../ext/net" }
|
||||
deno_tls = { version = "0.45.0", path = "../ext/tls" }
|
||||
deno_url = { version = "0.58.0", path = "../ext/url" }
|
||||
deno_web = { version = "0.89.0", path = "../ext/web" }
|
||||
deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" }
|
||||
deno_webidl = { version = "0.58.0", path = "../ext/webidl" }
|
||||
deno_websocket = { version = "0.63.0", path = "../ext/websocket" }
|
||||
deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" }
|
||||
|
||||
lzzzz = '1.0'
|
||||
|
||||
[target.'cfg(windows)'.build-dependencies]
|
||||
winres = "0.1.11"
|
||||
winapi = "0.3.9"
|
||||
|
|
222
runtime/build.rs
222
runtime/build.rs
|
@ -1,219 +1,17 @@
|
|||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
// This is a shim that allows to generate documentation on docs.rs
|
||||
#[cfg(not(feature = "docsrs"))]
|
||||
mod not_docs {
|
||||
use super::*;
|
||||
use deno_core::Extension;
|
||||
use deno_core::JsRuntime;
|
||||
use deno_core::RuntimeOptions;
|
||||
|
||||
// TODO(bartlomieju): this module contains a lot of duplicated
|
||||
// logic with `cli/build.rs`, factor out to `deno_core`.
|
||||
fn create_snapshot(
|
||||
mut js_runtime: JsRuntime,
|
||||
snapshot_path: &Path,
|
||||
files: Vec<PathBuf>,
|
||||
) {
|
||||
// TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the
|
||||
// workspace root.
|
||||
let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
|
||||
for file in files {
|
||||
println!("cargo:rerun-if-changed={}", file.display());
|
||||
let display_path = file.strip_prefix(display_root).unwrap();
|
||||
let display_path_str = display_path.display().to_string();
|
||||
js_runtime
|
||||
.execute_script(
|
||||
&("deno:".to_string() + &display_path_str.replace('\\', "/")),
|
||||
&std::fs::read_to_string(&file).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let snapshot = js_runtime.snapshot();
|
||||
let snapshot_slice: &[u8] = &*snapshot;
|
||||
println!("Snapshot size: {}", snapshot_slice.len());
|
||||
|
||||
let compressed_snapshot_with_size = {
|
||||
let mut vec = vec![];
|
||||
|
||||
vec.extend_from_slice(
|
||||
&u32::try_from(snapshot.len())
|
||||
.expect("snapshot larger than 4gb")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
|
||||
lzzzz::lz4_hc::compress_to_vec(
|
||||
snapshot_slice,
|
||||
&mut vec,
|
||||
lzzzz::lz4_hc::CLEVEL_MAX,
|
||||
)
|
||||
.expect("snapshot compression failed");
|
||||
|
||||
vec
|
||||
};
|
||||
|
||||
println!(
|
||||
"Snapshot compressed size: {}",
|
||||
compressed_snapshot_with_size.len()
|
||||
);
|
||||
|
||||
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
|
||||
println!("Snapshot written to: {} ", snapshot_path.display());
|
||||
}
|
||||
|
||||
struct Permissions;
|
||||
|
||||
impl deno_fetch::FetchPermissions for Permissions {
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_read(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_websocket::WebSocketPermissions for Permissions {
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_web::TimersPermission for Permissions {
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_unstable(
|
||||
&self,
|
||||
_state: &deno_core::OpState,
|
||||
_api_name: &'static str,
|
||||
) {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_ffi::FfiPermissions for Permissions {
|
||||
fn check(
|
||||
&mut self,
|
||||
_path: Option<&Path>,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_net::NetPermissions for Permissions {
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
_host: &(T, Option<u16>),
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_read(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_write(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
|
||||
let extensions: Vec<Extension> = vec![
|
||||
deno_webidl::init(),
|
||||
deno_console::init(),
|
||||
deno_url::init(),
|
||||
deno_tls::init(),
|
||||
deno_web::init::<Permissions>(
|
||||
deno_web::BlobStore::default(),
|
||||
Default::default(),
|
||||
),
|
||||
deno_fetch::init::<Permissions>(Default::default()),
|
||||
deno_websocket::init::<Permissions>("".to_owned(), None, None),
|
||||
deno_webstorage::init(None),
|
||||
deno_crypto::init(None),
|
||||
deno_webgpu::init(false),
|
||||
deno_broadcast_channel::init(
|
||||
deno_broadcast_channel::InMemoryBroadcastChannel::default(),
|
||||
false, // No --unstable.
|
||||
),
|
||||
deno_ffi::init::<Permissions>(false),
|
||||
deno_net::init::<Permissions>(
|
||||
None, false, // No --unstable.
|
||||
None,
|
||||
),
|
||||
deno_http::init(),
|
||||
];
|
||||
|
||||
let js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
will_snapshot: true,
|
||||
extensions,
|
||||
..Default::default()
|
||||
});
|
||||
create_snapshot(js_runtime, snapshot_path, files);
|
||||
}
|
||||
|
||||
fn get_js_files(d: &str) -> Vec<PathBuf> {
|
||||
let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut js_files = std::fs::read_dir(d)
|
||||
.unwrap()
|
||||
.map(|dir_entry| {
|
||||
let file = dir_entry.unwrap();
|
||||
manifest_dir.join(file.path())
|
||||
})
|
||||
.filter(|path| path.extension().unwrap_or_default() == "js")
|
||||
.collect::<Vec<PathBuf>>();
|
||||
js_files.sort();
|
||||
js_files
|
||||
}
|
||||
|
||||
pub fn build_snapshot(runtime_snapshot_path: PathBuf) {
|
||||
let js_files = get_js_files("js");
|
||||
create_runtime_snapshot(&runtime_snapshot_path, js_files);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// To debug snapshot issues uncomment:
|
||||
// op_fetch_asset::trace_serializer();
|
||||
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
|
||||
|
||||
// Main snapshot
|
||||
let runtime_snapshot_path = o.join("CLI_SNAPSHOT.bin");
|
||||
|
||||
// If we're building on docs.rs we just create
|
||||
// and empty snapshot file and return, because `rusty_v8`
|
||||
// doesn't actually compile on docs.rs
|
||||
if env::var_os("DOCS_RS").is_some() {
|
||||
let snapshot_slice = &[];
|
||||
std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap();
|
||||
// Skip building from docs.rs.
|
||||
if std::env::var_os("DOCS_RS").is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "docsrs"))]
|
||||
not_docs::build_snapshot(runtime_snapshot_path)
|
||||
println!(
|
||||
"cargo:rustc-env=TARGET={}",
|
||||
std::env::var("TARGET").unwrap()
|
||||
);
|
||||
println!(
|
||||
"cargo:rustc-env=PROFILE={}",
|
||||
std::env::var("PROFILE").unwrap()
|
||||
);
|
||||
}
|
||||
|
|
|
@ -58,6 +58,7 @@ async fn main() -> Result<(), AnyError> {
|
|||
shared_array_buffer_store: None,
|
||||
compiled_wasm_module_store: None,
|
||||
stdio: Default::default(),
|
||||
startup_snapshot: None,
|
||||
};
|
||||
|
||||
let js_path =
|
||||
|
|
|
@ -1,58 +1,38 @@
|
|||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
use deno_core::Snapshot;
|
||||
use log::debug;
|
||||
use once_cell::sync::Lazy;
|
||||
use deno_core::include_js_files;
|
||||
use deno_core::Extension;
|
||||
|
||||
pub static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
|
||||
#[allow(clippy::uninit_vec)]
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
|| {
|
||||
static COMPRESSED_CLI_SNAPSHOT: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
|
||||
|
||||
let size =
|
||||
u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
|
||||
as usize;
|
||||
let mut vec = Vec::with_capacity(size);
|
||||
|
||||
// SAFETY: vec is allocated with exact snapshot size (+ alignment)
|
||||
// SAFETY: non zeroed bytes are overwritten with decompressed snapshot
|
||||
unsafe {
|
||||
vec.set_len(size);
|
||||
}
|
||||
|
||||
lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
|
||||
|
||||
vec.into_boxed_slice()
|
||||
},
|
||||
);
|
||||
|
||||
pub fn deno_isolate_init() -> Snapshot {
|
||||
debug!("Deno isolate init with snapshots.");
|
||||
Snapshot::Static(&*CLI_SNAPSHOT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cli_snapshot() {
|
||||
let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
|
||||
startup_snapshot: Some(deno_isolate_init()),
|
||||
..Default::default()
|
||||
});
|
||||
js_runtime
|
||||
.execute_script(
|
||||
"<anon>",
|
||||
r#"
|
||||
if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) {
|
||||
throw Error("bad");
|
||||
}
|
||||
console.log("we have console.log!!!");
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
pub fn init() -> Extension {
|
||||
Extension::builder()
|
||||
.js(include_js_files!(
|
||||
prefix "deno:runtime",
|
||||
// Generated with:
|
||||
// bash -c "cd runtime && ls js/*.js | sort"
|
||||
"js/01_build.js",
|
||||
"js/01_errors.js",
|
||||
"js/01_version.js",
|
||||
"js/01_web_util.js",
|
||||
"js/06_util.js",
|
||||
"js/10_permissions.js",
|
||||
"js/11_workers.js",
|
||||
"js/12_io.js",
|
||||
"js/13_buffer.js",
|
||||
"js/30_fs.js",
|
||||
"js/30_os.js",
|
||||
"js/40_diagnostics.js",
|
||||
"js/40_files.js",
|
||||
"js/40_fs_events.js",
|
||||
"js/40_http.js",
|
||||
"js/40_process.js",
|
||||
"js/40_read_file.js",
|
||||
"js/40_signals.js",
|
||||
"js/40_spawn.js",
|
||||
"js/40_testing.js",
|
||||
"js/40_tty.js",
|
||||
"js/40_write_file.js",
|
||||
"js/41_prompt.js",
|
||||
"js/90_deno_ns.js",
|
||||
"js/99_main.js",
|
||||
))
|
||||
.build()
|
||||
}
|
||||
|
|
|
@ -335,6 +335,7 @@ pub struct WebWorkerOptions {
|
|||
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
|
||||
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
|
||||
pub stdio: Stdio,
|
||||
pub startup_snapshot: Option<deno_core::Snapshot>,
|
||||
}
|
||||
|
||||
impl WebWorker {
|
||||
|
@ -427,6 +428,8 @@ impl WebWorker {
|
|||
ops::tty::init(),
|
||||
deno_http::init(),
|
||||
ops::http::init(),
|
||||
// Runtime JS
|
||||
js::init(),
|
||||
// Permissions ext (worker specific state)
|
||||
perm_ext,
|
||||
];
|
||||
|
@ -436,7 +439,7 @@ impl WebWorker {
|
|||
|
||||
let mut js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
module_loader: Some(options.module_loader.clone()),
|
||||
startup_snapshot: Some(js::deno_isolate_init()),
|
||||
startup_snapshot: options.startup_snapshot.take(),
|
||||
source_map_getter: options.source_map_getter,
|
||||
get_error_class_fn: options.get_error_class_fn,
|
||||
shared_array_buffer_store: options.shared_array_buffer_store.clone(),
|
||||
|
|
|
@ -81,6 +81,7 @@ pub struct WorkerOptions {
|
|||
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
|
||||
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
|
||||
pub stdio: Stdio,
|
||||
pub startup_snapshot: Option<deno_core::Snapshot>,
|
||||
}
|
||||
|
||||
impl MainWorker {
|
||||
|
@ -168,6 +169,8 @@ impl MainWorker {
|
|||
ops::tty::init(),
|
||||
deno_http::init(),
|
||||
ops::http::init(),
|
||||
// Runtime JS
|
||||
js::init(),
|
||||
// Permissions ext (worker specific state)
|
||||
perm_ext,
|
||||
];
|
||||
|
@ -175,7 +178,7 @@ impl MainWorker {
|
|||
|
||||
let mut js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
module_loader: Some(options.module_loader.clone()),
|
||||
startup_snapshot: Some(js::deno_isolate_init()),
|
||||
startup_snapshot: options.startup_snapshot.take(),
|
||||
source_map_getter: options.source_map_getter,
|
||||
get_error_class_fn: options.get_error_class_fn,
|
||||
shared_array_buffer_store: options.shared_array_buffer_store.clone(),
|
||||
|
@ -411,6 +414,7 @@ mod tests {
|
|||
shared_array_buffer_store: None,
|
||||
compiled_wasm_module_store: None,
|
||||
stdio: Default::default(),
|
||||
startup_snapshot: None,
|
||||
};
|
||||
|
||||
MainWorker::bootstrap_from_options(main_module, permissions, options)
|
||||
|
|
32
snapshots/Cargo.toml
Normal file
32
snapshots/Cargo.toml
Normal file
|
@ -0,0 +1,32 @@
|
|||
# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
# IMPORTANT(bartlomieju): this crate is internal and shouldn't be published
|
||||
# to crates.io
|
||||
|
||||
[package]
|
||||
name = "deno_snapshots"
|
||||
version = "0.0.0"
|
||||
authors = ["the Deno authors"]
|
||||
edition = "2018"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/denoland/deno"
|
||||
description = "Provides snapshots of TSC & Deno (runtime+web+core)"
|
||||
|
||||
[lib]
|
||||
name = "deno_snapshots"
|
||||
path = "lib.rs"
|
||||
|
||||
[dependencies]
|
||||
deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
|
||||
deno_runtime = { version = "0.66.0", path = "../runtime" }
|
||||
lzzzz = "1.0"
|
||||
once_cell = "1.10.0"
|
||||
zstd = "0.11.1"
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
|
||||
deno_runtime = { version = "0.66.0", path = "../runtime" }
|
||||
lzzzz = "1.0"
|
||||
regex = "1.5.6"
|
||||
serde = { version = "1.0.125", features = ["derive"] }
|
||||
zstd = "0.11.1"
|
10
snapshots/build.rs
Normal file
10
snapshots/build.rs
Normal file
|
@ -0,0 +1,10 @@
|
|||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
mod build_runtime;
|
||||
mod build_tsc;
|
||||
|
||||
fn main() {
|
||||
let out_dir = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
|
||||
build_runtime::create_runtime_snapshot(&out_dir.join("CLI_SNAPSHOT.bin"));
|
||||
build_tsc::create_tsc_snapshot(&out_dir.join("COMPILER_SNAPSHOT.bin"));
|
||||
}
|
168
snapshots/build_runtime.rs
Normal file
168
snapshots/build_runtime.rs
Normal file
|
@ -0,0 +1,168 @@
|
|||
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
|
||||
use std::convert::TryFrom;
|
||||
use std::path::Path;
|
||||
|
||||
use deno_runtime::deno_broadcast_channel;
|
||||
use deno_runtime::deno_console;
|
||||
use deno_runtime::deno_core;
|
||||
use deno_runtime::deno_crypto;
|
||||
use deno_runtime::deno_fetch;
|
||||
use deno_runtime::deno_ffi;
|
||||
use deno_runtime::deno_http;
|
||||
use deno_runtime::deno_net;
|
||||
use deno_runtime::deno_tls;
|
||||
use deno_runtime::deno_url;
|
||||
use deno_runtime::deno_web;
|
||||
use deno_runtime::deno_webgpu;
|
||||
use deno_runtime::deno_webidl;
|
||||
use deno_runtime::deno_websocket;
|
||||
use deno_runtime::deno_webstorage;
|
||||
|
||||
use deno_core::Extension;
|
||||
use deno_core::JsRuntime;
|
||||
use deno_core::RuntimeOptions;
|
||||
|
||||
pub fn create_runtime_snapshot(snapshot_path: &Path) {
|
||||
let extensions: Vec<Extension> = vec![
|
||||
deno_webidl::init(),
|
||||
deno_console::init(),
|
||||
deno_url::init(),
|
||||
deno_tls::init(),
|
||||
deno_web::init::<Permissions>(
|
||||
deno_web::BlobStore::default(),
|
||||
Default::default(),
|
||||
),
|
||||
deno_fetch::init::<Permissions>(Default::default()),
|
||||
deno_websocket::init::<Permissions>("".to_owned(), None, None),
|
||||
deno_webstorage::init(None),
|
||||
deno_crypto::init(None),
|
||||
deno_webgpu::init(false),
|
||||
deno_broadcast_channel::init(
|
||||
deno_broadcast_channel::InMemoryBroadcastChannel::default(),
|
||||
false, // No --unstable.
|
||||
),
|
||||
deno_ffi::init::<Permissions>(false),
|
||||
deno_net::init::<Permissions>(
|
||||
None, false, // No --unstable.
|
||||
None,
|
||||
),
|
||||
deno_http::init(),
|
||||
// Runtime JS
|
||||
deno_runtime::js::init(),
|
||||
];
|
||||
|
||||
let js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
will_snapshot: true,
|
||||
extensions,
|
||||
..Default::default()
|
||||
});
|
||||
write_runtime_snapshot(js_runtime, snapshot_path);
|
||||
}
|
||||
|
||||
// TODO(bartlomieju): this module contains a lot of duplicated
|
||||
// logic with `build_tsc.rs`
|
||||
fn write_runtime_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
|
||||
let snapshot = js_runtime.snapshot();
|
||||
let snapshot_slice: &[u8] = &*snapshot;
|
||||
println!("Snapshot size: {}", snapshot_slice.len());
|
||||
|
||||
let compressed_snapshot_with_size = {
|
||||
let mut vec = vec![];
|
||||
|
||||
vec.extend_from_slice(
|
||||
&u32::try_from(snapshot.len())
|
||||
.expect("snapshot larger than 4gb")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
|
||||
lzzzz::lz4_hc::compress_to_vec(
|
||||
snapshot_slice,
|
||||
&mut vec,
|
||||
lzzzz::lz4_hc::CLEVEL_MAX,
|
||||
)
|
||||
.expect("snapshot compression failed");
|
||||
|
||||
vec
|
||||
};
|
||||
|
||||
println!(
|
||||
"Snapshot compressed size: {}",
|
||||
compressed_snapshot_with_size.len()
|
||||
);
|
||||
|
||||
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
|
||||
println!("Snapshot written to: {} ", snapshot_path.display());
|
||||
}
|
||||
|
||||
struct Permissions;
|
||||
|
||||
impl deno_fetch::FetchPermissions for Permissions {
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_read(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_websocket::WebSocketPermissions for Permissions {
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_web::TimersPermission for Permissions {
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_unstable(
|
||||
&self,
|
||||
_state: &deno_core::OpState,
|
||||
_api_name: &'static str,
|
||||
) {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_ffi::FfiPermissions for Permissions {
|
||||
fn check(
|
||||
&mut self,
|
||||
_path: Option<&Path>,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
||||
|
||||
impl deno_net::NetPermissions for Permissions {
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
_host: &(T, Option<u16>),
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_read(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
|
||||
fn check_write(
|
||||
&mut self,
|
||||
_p: &Path,
|
||||
) -> Result<(), deno_core::error::AnyError> {
|
||||
unreachable!("snapshotting!")
|
||||
}
|
||||
}
|
328
snapshots/build_tsc.rs
Normal file
328
snapshots/build_tsc.rs
Normal file
|
@ -0,0 +1,328 @@
|
|||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
use deno_runtime::deno_broadcast_channel;
|
||||
use deno_runtime::deno_console;
|
||||
use deno_runtime::deno_crypto;
|
||||
use deno_runtime::deno_fetch;
|
||||
use deno_runtime::deno_net;
|
||||
use deno_runtime::deno_url;
|
||||
use deno_runtime::deno_web;
|
||||
use deno_runtime::deno_websocket;
|
||||
use deno_runtime::deno_webstorage;
|
||||
|
||||
use deno_runtime::deno_core::error::custom_error;
|
||||
use deno_runtime::deno_core::error::AnyError;
|
||||
use deno_runtime::deno_core::op;
|
||||
use deno_runtime::deno_core::serde::Deserialize;
|
||||
use deno_runtime::deno_core::serde_json::json;
|
||||
use deno_runtime::deno_core::serde_json::Value;
|
||||
use deno_runtime::deno_core::Extension;
|
||||
use deno_runtime::deno_core::JsRuntime;
|
||||
use deno_runtime::deno_core::OpState;
|
||||
use deno_runtime::deno_core::RuntimeOptions;
|
||||
|
||||
use regex::Regex;
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryFrom;
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub fn create_tsc_snapshot(snapshot_path: &Path) {
|
||||
let mut js_runtime = JsRuntime::new(RuntimeOptions {
|
||||
will_snapshot: true,
|
||||
extensions: vec![tsc_snapshot_init()],
|
||||
..Default::default()
|
||||
});
|
||||
load_js_files(&mut js_runtime);
|
||||
write_snapshot(js_runtime, snapshot_path);
|
||||
}
|
||||
|
||||
// TODO(bartlomieju): this module contains a lot of duplicated
|
||||
// logic with `build_runtime.rs`
|
||||
fn write_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
|
||||
let snapshot = js_runtime.snapshot();
|
||||
let snapshot_slice: &[u8] = &*snapshot;
|
||||
println!("Snapshot size: {}", snapshot_slice.len());
|
||||
|
||||
let compressed_snapshot_with_size = {
|
||||
let mut vec = vec![];
|
||||
|
||||
vec.extend_from_slice(
|
||||
&u32::try_from(snapshot.len())
|
||||
.expect("snapshot larger than 4gb")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
|
||||
vec.extend_from_slice(
|
||||
&zstd::bulk::compress(snapshot_slice, 22)
|
||||
.expect("snapshot compression failed"),
|
||||
);
|
||||
|
||||
vec
|
||||
};
|
||||
|
||||
println!(
|
||||
"Snapshot compressed size: {}",
|
||||
compressed_snapshot_with_size.len()
|
||||
);
|
||||
|
||||
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
|
||||
println!("Snapshot written to: {} ", snapshot_path.display());
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LoadArgs {
|
||||
/// The fully qualified specifier that should be loaded.
|
||||
specifier: String,
|
||||
}
|
||||
|
||||
fn tsc_snapshot_init() -> Extension {
|
||||
// libs that are being provided by op crates.
|
||||
let mut op_crate_libs = HashMap::new();
|
||||
op_crate_libs.insert("deno.console", deno_console::get_declaration());
|
||||
op_crate_libs.insert("deno.url", deno_url::get_declaration());
|
||||
op_crate_libs.insert("deno.web", deno_web::get_declaration());
|
||||
op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
|
||||
op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration());
|
||||
op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration());
|
||||
op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration());
|
||||
op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration());
|
||||
op_crate_libs.insert(
|
||||
"deno.broadcast_channel",
|
||||
deno_broadcast_channel::get_declaration(),
|
||||
);
|
||||
op_crate_libs.insert("deno.net", deno_net::get_declaration());
|
||||
|
||||
// ensure we invalidate the build properly.
|
||||
for (_, path) in op_crate_libs.iter() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
|
||||
// libs that should be loaded into the isolate before snapshotting.
|
||||
let libs = vec![
|
||||
// Deno custom type libraries
|
||||
"deno.window",
|
||||
"deno.worker",
|
||||
"deno.shared_globals",
|
||||
"deno.ns",
|
||||
"deno.unstable",
|
||||
// Deno built-in type libraries
|
||||
"es5",
|
||||
"es2015.collection",
|
||||
"es2015.core",
|
||||
"es2015",
|
||||
"es2015.generator",
|
||||
"es2015.iterable",
|
||||
"es2015.promise",
|
||||
"es2015.proxy",
|
||||
"es2015.reflect",
|
||||
"es2015.symbol",
|
||||
"es2015.symbol.wellknown",
|
||||
"es2016.array.include",
|
||||
"es2016",
|
||||
"es2017",
|
||||
"es2017.intl",
|
||||
"es2017.object",
|
||||
"es2017.sharedmemory",
|
||||
"es2017.string",
|
||||
"es2017.typedarrays",
|
||||
"es2018.asyncgenerator",
|
||||
"es2018.asynciterable",
|
||||
"es2018",
|
||||
"es2018.intl",
|
||||
"es2018.promise",
|
||||
"es2018.regexp",
|
||||
"es2019.array",
|
||||
"es2019",
|
||||
"es2019.object",
|
||||
"es2019.string",
|
||||
"es2019.symbol",
|
||||
"es2020.bigint",
|
||||
"es2020",
|
||||
"es2020.date",
|
||||
"es2020.intl",
|
||||
"es2020.number",
|
||||
"es2020.promise",
|
||||
"es2020.sharedmemory",
|
||||
"es2020.string",
|
||||
"es2020.symbol.wellknown",
|
||||
"es2021",
|
||||
"es2021.intl",
|
||||
"es2021.promise",
|
||||
"es2021.string",
|
||||
"es2021.weakref",
|
||||
"es2022",
|
||||
"es2022.array",
|
||||
"es2022.error",
|
||||
"es2022.intl",
|
||||
"es2022.object",
|
||||
"es2022.string",
|
||||
"esnext",
|
||||
"esnext.array",
|
||||
"esnext.intl",
|
||||
];
|
||||
|
||||
let cli_dir = cli_dir();
|
||||
let path_dts = cli_dir.join("dts");
|
||||
// ensure we invalidate the build properly.
|
||||
for name in libs.iter() {
|
||||
println!(
|
||||
"cargo:rerun-if-changed={}",
|
||||
path_dts.join(format!("lib.{}.d.ts", name)).display()
|
||||
);
|
||||
}
|
||||
|
||||
// create a copy of the vector that includes any op crate libs to be passed
|
||||
// to the JavaScript compiler to build into the snapshot
|
||||
let mut build_libs = libs.clone();
|
||||
for (op_lib, _) in op_crate_libs.iter() {
|
||||
build_libs.push(op_lib.to_owned());
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_build_info(state: &mut OpState) -> Value {
|
||||
let build_specifier = "asset:///bootstrap.ts";
|
||||
let build_libs = state.borrow::<Vec<&str>>();
|
||||
json!({
|
||||
"buildSpecifier": build_specifier,
|
||||
"libs": build_libs,
|
||||
})
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_cwd() -> String {
|
||||
"cache:///".into()
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_exists() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_script_version(
|
||||
_state: &mut OpState,
|
||||
_args: Value,
|
||||
) -> Result<Option<String>, AnyError> {
|
||||
Ok(Some("1".to_string()))
|
||||
}
|
||||
|
||||
#[op]
|
||||
// using the same op that is used in `tsc.rs` for loading modules and reading
|
||||
// files, but a slightly different implementation at build time.
|
||||
fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> {
|
||||
let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>();
|
||||
let path_dts = state.borrow::<PathBuf>();
|
||||
let re_asset =
|
||||
Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
|
||||
let build_specifier = "asset:///bootstrap.ts";
|
||||
|
||||
// we need a basic file to send to tsc to warm it up.
|
||||
if args.specifier == build_specifier {
|
||||
Ok(json!({
|
||||
"data": r#"console.log("hello deno!");"#,
|
||||
"version": "1",
|
||||
// this corresponds to `ts.ScriptKind.TypeScript`
|
||||
"scriptKind": 3
|
||||
}))
|
||||
// specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to
|
||||
// parse out just the name so we can lookup the asset.
|
||||
} else if let Some(caps) = re_asset.captures(&args.specifier) {
|
||||
if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
|
||||
// if it comes from an op crate, we were supplied with the path to the
|
||||
// file.
|
||||
let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
|
||||
PathBuf::from(op_crate_lib).canonicalize().unwrap()
|
||||
// otherwise we are will generate the path ourself
|
||||
} else {
|
||||
path_dts.join(format!("lib.{}.d.ts", lib))
|
||||
};
|
||||
let data = std::fs::read_to_string(path)?;
|
||||
Ok(json!({
|
||||
"data": data,
|
||||
"version": "1",
|
||||
// this corresponds to `ts.ScriptKind.TypeScript`
|
||||
"scriptKind": 3
|
||||
}))
|
||||
} else {
|
||||
Err(custom_error(
|
||||
"InvalidSpecifier",
|
||||
format!("An invalid specifier was requested: {}", args.specifier),
|
||||
))
|
||||
}
|
||||
} else {
|
||||
Err(custom_error(
|
||||
"InvalidSpecifier",
|
||||
format!("An invalid specifier was requested: {}", args.specifier),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
Extension::builder()
|
||||
.ops(vec![
|
||||
op_build_info::decl(),
|
||||
op_cwd::decl(),
|
||||
op_exists::decl(),
|
||||
op_load::decl(),
|
||||
op_script_version::decl(),
|
||||
])
|
||||
.state(move |state| {
|
||||
state.put(op_crate_libs.clone());
|
||||
state.put(build_libs.clone());
|
||||
state.put(path_dts.clone());
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.build()
|
||||
}
|
||||
|
||||
fn deno_webgpu_get_declaration() -> PathBuf {
|
||||
cli_dir().join("dts").join("lib.deno_webgpu.d.ts")
|
||||
}
|
||||
|
||||
fn load_js_files(js_runtime: &mut JsRuntime) {
|
||||
let js_files = get_js_files(tsc_dir());
|
||||
let cwd = cli_dir();
|
||||
let display_root = cwd.parent().unwrap();
|
||||
for file in js_files {
|
||||
println!("cargo:rerun-if-changed={}", file.display());
|
||||
let display_path = file.strip_prefix(display_root).unwrap();
|
||||
let display_path_str = display_path.display().to_string();
|
||||
js_runtime
|
||||
.execute_script(
|
||||
&("deno:".to_string() + &display_path_str.replace('\\', "/")),
|
||||
&std::fs::read_to_string(&file).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn root_dir() -> PathBuf {
|
||||
// TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the workspace root.
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("..")
|
||||
.canonicalize()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn cli_dir() -> PathBuf {
|
||||
root_dir().join("cli")
|
||||
}
|
||||
|
||||
fn tsc_dir() -> PathBuf {
|
||||
cli_dir().join("tsc")
|
||||
}
|
||||
|
||||
fn get_js_files(dir: PathBuf) -> Vec<PathBuf> {
|
||||
let mut js_files = std::fs::read_dir(dir.clone())
|
||||
.unwrap()
|
||||
.map(|dir_entry| {
|
||||
let file = dir_entry.unwrap();
|
||||
dir.join(file.path())
|
||||
})
|
||||
.filter(|path| path.extension().unwrap_or_default() == "js")
|
||||
.collect::<Vec<PathBuf>>();
|
||||
js_files.sort();
|
||||
js_files
|
||||
}
|
77
snapshots/lib.rs
Normal file
77
snapshots/lib.rs
Normal file
|
@ -0,0 +1,77 @@
|
|||
use deno_core::Snapshot;
|
||||
use once_cell::sync::Lazy;
|
||||
use std::convert::TryInto;
|
||||
|
||||
pub fn tsc_snapshot() -> Snapshot {
|
||||
Snapshot::Static(&*COMPILER_SNAPSHOT)
|
||||
}
|
||||
|
||||
static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
|| {
|
||||
static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
|
||||
|
||||
zstd::bulk::decompress(
|
||||
&COMPRESSED_COMPILER_SNAPSHOT[4..],
|
||||
u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
|
||||
as usize,
|
||||
)
|
||||
.unwrap()
|
||||
.into_boxed_slice()
|
||||
},
|
||||
);
|
||||
|
||||
pub fn cli_snapshot() -> Snapshot {
|
||||
Snapshot::Static(&*CLI_SNAPSHOT)
|
||||
}
|
||||
|
||||
static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
|
||||
#[allow(clippy::uninit_vec)]
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
|| {
|
||||
static COMPRESSED_CLI_SNAPSHOT: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
|
||||
|
||||
let size =
|
||||
u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
|
||||
as usize;
|
||||
let mut vec = Vec::with_capacity(size);
|
||||
|
||||
// SAFETY: vec is allocated with exact snapshot size (+ alignment)
|
||||
// SAFETY: non zeroed bytes are overwritten with decompressed snapshot
|
||||
unsafe {
|
||||
vec.set_len(size);
|
||||
}
|
||||
|
||||
lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
|
||||
|
||||
vec.into_boxed_slice()
|
||||
},
|
||||
);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use deno_runtime::deno_core;
|
||||
|
||||
#[test]
|
||||
fn cli_snapshot() {
|
||||
let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
|
||||
startup_snapshot: Some(crate::cli_snapshot()),
|
||||
..Default::default()
|
||||
});
|
||||
js_runtime
|
||||
.execute_script(
|
||||
"<anon>",
|
||||
r#"
|
||||
if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) {
|
||||
throw Error("bad");
|
||||
}
|
||||
console.log("we have console.log!!!");
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue