refactor(runtime): factor out code between build.rs and worker.rs (#20299)

Adds `runtime/shared.rs` which is imported by both `build.rs` and the
rest of the crate, containing utilities used by both.
Renames the `snapshot_from_snapshot` feature to
`exclude_runtime_main_js` since that's what it does and it's relevant
outside of snapshotting when `__runtime_js_sources` is specified.

cli/Cargo.toml

@@ -32,7 +32,7 @@ path = "./bench/lsp_bench_standalone.rs"
 __runtime_js_sources = ["deno_runtime/__runtime_js_sources"]
 
 [build-dependencies]
-deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] }
+deno_runtime = { workspace = true, features = ["exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 lazy-regex.workspace = true
 serde.workspace = true
@@ -55,7 +55,7 @@ deno_graph = "=0.52.0"
 deno_lint = { version = "=0.50.2", features = ["docs"] }
 deno_lockfile.workspace = true
 deno_npm.workspace = true
-deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
 deno_semver.workspace = true
 deno_task_shell = "=0.13.2"
 eszip = "=0.50.0"

runtime/Cargo.toml

@@ -14,9 +14,8 @@ description = "Provides the deno runtime library"
 docsrs = []
 # A feature that disables creation of startup snapshot during in the build script.
 dont_create_runtime_snapshot = []
-# A feature that changes how startup snapshot is generated, that allows
-# extending it in embedder crates.
-snapshot_from_snapshot = []
+# A feature that allows excluding `./js/99_main.js` from the exported extension.
+exclude_runtime_main_js = []
 # A feature that disables embedding of the JavaScript source files in the binary.
 # With this feature enabled, the sources must be consumed during build time,
 # by creating a startup snapshot.

runtime/build.rs

@@ -1,5 +1,7 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
+mod shared;
+
 use std::env;
 use std::path::PathBuf;
 
@@ -9,59 +11,15 @@ use std::path::PathBuf;
 ))]
 mod startup_snapshot {
   use super::*;
-  use deno_ast::MediaType;
-  use deno_ast::ParseParams;
-  use deno_ast::SourceTextInfo;
   use deno_cache::SqliteBackedCache;
   use deno_core::error::AnyError;
   use deno_core::snapshot_util::*;
   use deno_core::Extension;
-  use deno_core::ExtensionFileSource;
-  use deno_core::ExtensionFileSourceCode;
   use deno_http::DefaultHttpPropertyExtractor;
+  use shared::maybe_transpile_source;
+  use shared::runtime;
   use std::path::Path;
 
-  // Duplicated in `worker.rs`. Keep in sync!
-  fn maybe_transpile_source(
-    source: &mut ExtensionFileSource,
-  ) -> Result<(), AnyError> {
-    // Always transpile `node:` built-in modules, since they might be TypeScript.
-    let media_type = if source.specifier.starts_with("node:") {
-      MediaType::TypeScript
-    } else {
-      MediaType::from_path(Path::new(&source.specifier))
-    };
-
-    match media_type {
-      MediaType::TypeScript => {}
-      MediaType::JavaScript => return Ok(()),
-      MediaType::Mjs => return Ok(()),
-      _ => panic!(
-        "Unsupported media type for snapshotting {media_type:?} for file {}",
-        source.specifier
-      ),
-    }
-    let code = source.load()?;
-
-    let parsed = deno_ast::parse_module(ParseParams {
-      specifier: source.specifier.to_string(),
-      text_info: SourceTextInfo::from_string(code.as_str().to_owned()),
-      media_type,
-      capture_tokens: false,
-      scope_analysis: false,
-      maybe_syntax: None,
-    })?;
-    let transpiled_source = parsed.transpile(&deno_ast::EmitOptions {
-      imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove,
-      inline_source_map: false,
-      ..Default::default()
-    })?;
-
-    source.code =
-      ExtensionFileSourceCode::Computed(transpiled_source.text.into());
-    Ok(())
-  }
-
   #[derive(Clone)]
   struct Permissions;
 
@@ -241,71 +199,6 @@ mod startup_snapshot {
     }
   }
 
-  // Duplicated in `worker.rs`. Keep in sync!
-  deno_core::extension!(runtime,
-    deps = [
-      deno_webidl,
-      deno_console,
-      deno_url,
-      deno_tls,
-      deno_web,
-      deno_fetch,
-      deno_cache,
-      deno_websocket,
-      deno_webstorage,
-      deno_crypto,
-      deno_broadcast_channel,
-      // FIXME(bartlomieju): this should be reenabled
-      // "deno_node",
-      deno_ffi,
-      deno_net,
-      deno_napi,
-      deno_http,
-      deno_io,
-      deno_fs
-    ],
-    esm = [
-      dir "js",
-      "01_errors.js",
-      "01_version.ts",
-      "06_util.js",
-      "10_permissions.js",
-      "11_workers.js",
-      "13_buffer.js",
-      "30_os.js",
-      "40_fs_events.js",
-      "40_http.js",
-      "40_process.js",
-      "40_signals.js",
-      "40_tty.js",
-      "41_prompt.js",
-      "90_deno_ns.js",
-      "98_global_scope.js"
-    ],
-  );
-
-  #[cfg(not(feature = "snapshot_from_snapshot"))]
-  deno_core::extension!(
-    runtime_main,
-    deps = [runtime],
-    esm_entry_point = "ext:runtime_main/js/99_main.js",
-    customizer = |ext: &mut deno_core::Extension| {
-      ext.esm_files.to_mut().push(ExtensionFileSource {
-        specifier: "ext:runtime_main/js/99_main.js",
-        code: deno_core::ExtensionFileSourceCode::IncludedInBinary(
-          include_str!("js/99_main.js"),
-        ),
-      });
-    }
-  );
-
-  #[cfg(feature = "snapshot_from_snapshot")]
-  deno_core::extension!(
-    runtime_main,
-    deps = [runtime],
-    esm_entry_point = "ext:runtime/90_deno_ns.js",
-  );
-
   pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
     // NOTE(bartlomieju): ordering is important here, keep it in sync with
     // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
@@ -347,11 +240,8 @@ mod startup_snapshot {
       deno_http::deno_http::init_ops_and_esm::<DefaultHttpPropertyExtractor>(),
       deno_io::deno_io::init_ops_and_esm(Default::default()),
       deno_fs::deno_fs::init_ops_and_esm::<Permissions>(false, fs.clone()),
-      runtime::init_ops_and_esm(),
-      // FIXME(bartlomieju): these extensions are specified last, because they
-      // depend on `runtime`, even though it should be other way around
       deno_node::deno_node::init_ops_and_esm::<Permissions>(None, fs),
-      runtime_main::init_ops_and_esm(),
+      runtime::init_ops_and_esm(),
     ];
 
     for extension in &mut extensions {

runtime/lib.rs

@@ -34,6 +34,8 @@ pub mod web_worker;
 pub mod worker;
 
 mod worker_bootstrap;
-pub use worker::runtime;
 pub use worker_bootstrap::BootstrapOptions;
 pub use worker_bootstrap::WorkerLogLevel;
+
+mod shared;
+pub use shared::runtime;

runtime/shared.rs

@@ -0,0 +1,107 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+// Utilities shared between `build.rs` and the rest of the crate.
+
+use deno_ast::MediaType;
+use deno_ast::ParseParams;
+use deno_ast::SourceTextInfo;
+use deno_core::error::AnyError;
+use deno_core::extension;
+use deno_core::Extension;
+use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
+use std::path::Path;
+
+extension!(runtime,
+  deps = [
+    deno_webidl,
+    deno_console,
+    deno_url,
+    deno_tls,
+    deno_web,
+    deno_fetch,
+    deno_cache,
+    deno_websocket,
+    deno_webstorage,
+    deno_crypto,
+    deno_broadcast_channel,
+    deno_node,
+    deno_ffi,
+    deno_net,
+    deno_napi,
+    deno_http,
+    deno_io,
+    deno_fs
+  ],
+  esm_entry_point = "ext:runtime/90_deno_ns.js",
+  esm = [
+    dir "js",
+    "01_errors.js",
+    "01_version.ts",
+    "06_util.js",
+    "10_permissions.js",
+    "11_workers.js",
+    "13_buffer.js",
+    "30_os.js",
+    "40_fs_events.js",
+    "40_http.js",
+    "40_process.js",
+    "40_signals.js",
+    "40_tty.js",
+    "41_prompt.js",
+    "90_deno_ns.js",
+    "98_global_scope.js"
+  ],
+  customizer = |ext: &mut Extension| {
+    #[cfg(not(feature = "exclude_runtime_main_js"))]
+    {
+      ext.esm_files.to_mut().push(ExtensionFileSource {
+        specifier: "ext:runtime_main/js/99_main.js",
+        code: ExtensionFileSourceCode::IncludedInBinary(
+          include_str!("./js/99_main.js"),
+        ),
+      });
+      ext.esm_entry_point = Some("ext:runtime_main/js/99_main.js");
+    }
+  }
+);
+
+#[allow(dead_code)]
+pub fn maybe_transpile_source(
+  source: &mut ExtensionFileSource,
+) -> Result<(), AnyError> {
+  // Always transpile `node:` built-in modules, since they might be TypeScript.
+  let media_type = if source.specifier.starts_with("node:") {
+    MediaType::TypeScript
+  } else {
+    MediaType::from_path(Path::new(&source.specifier))
+  };
+
+  match media_type {
+    MediaType::TypeScript => {}
+    MediaType::JavaScript => return Ok(()),
+    MediaType::Mjs => return Ok(()),
+    _ => panic!(
+      "Unsupported media type for snapshotting {media_type:?} for file {}",
+      source.specifier
+    ),
+  }
+  let code = source.load()?;
+
+  let parsed = deno_ast::parse_module(ParseParams {
+    specifier: source.specifier.to_string(),
+    text_info: SourceTextInfo::from_string(code.as_str().to_owned()),
+    media_type,
+    capture_tokens: false,
+    scope_analysis: false,
+    maybe_syntax: None,
+  })?;
+  let transpiled_source = parsed.transpile(&deno_ast::EmitOptions {
+    imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove,
+    inline_source_map: false,
+    ..Default::default()
+  })?;
+
+  source.code =
+    ExtensionFileSourceCode::Computed(transpiled_source.text.into());
+  Ok(())
+}

runtime/web_worker.rs

@@ -3,8 +3,8 @@ use crate::colors;
 use crate::inspector_server::InspectorServer;
 use crate::ops;
 use crate::permissions::PermissionsContainer;
+use crate::shared::runtime;
 use crate::tokio_util::create_and_run_current_thread;
-use crate::worker::runtime;
 use crate::worker::FormatJsErrorFn;
 use crate::BootstrapOptions;
 use deno_broadcast_channel::InMemoryBroadcastChannel;
@@ -484,7 +484,7 @@ impl WebWorker {
       }
       #[cfg(feature = "__runtime_js_sources")]
       {
-        use crate::worker::maybe_transpile_source;
+        use crate::shared::maybe_transpile_source;
         for source in extension.esm_files.to_mut() {
           maybe_transpile_source(source).unwrap();
         }

runtime/worker.rs

@@ -18,8 +18,6 @@ use deno_core::futures::Future;
 use deno_core::v8;
 use deno_core::CompiledWasmModuleStore;
 use deno_core::Extension;
-#[cfg(feature = "__runtime_js_sources")]
-use deno_core::ExtensionFileSource;
 use deno_core::FsModuleLoader;
 use deno_core::GetErrorClassFn;
 use deno_core::JsRuntime;
@@ -44,6 +42,7 @@ use log::debug;
 use crate::inspector_server::InspectorServer;
 use crate::ops;
 use crate::permissions::PermissionsContainer;
+use crate::shared::runtime;
 use crate::BootstrapOptions;
 
 pub type FormatJsErrorFn = dyn Fn(&JsError) -> String + Sync + Send;
@@ -61,78 +60,6 @@ impl ExitCode {
   }
 }
 
-// Duplicated in `build.rs`. Keep in sync!
-deno_core::extension!(
-  runtime,
-  esm_entry_point = "ext:runtime/90_deno_ns.js",
-  esm = [
-    dir "js",
-    "01_errors.js",
-    "01_version.ts",
-    "06_util.js",
-    "10_permissions.js",
-    "11_workers.js",
-    "13_buffer.js",
-    "30_os.js",
-    "40_fs_events.js",
-    "40_http.js",
-    "40_process.js",
-    "40_signals.js",
-    "40_tty.js",
-    "41_prompt.js",
-    "90_deno_ns.js",
-    "98_global_scope.js"
-  ],
-);
-
-// Duplicated in `build.rs`. Keep in sync!
-#[cfg(feature = "__runtime_js_sources")]
-pub fn maybe_transpile_source(
-  source: &mut ExtensionFileSource,
-) -> Result<(), AnyError> {
-  use deno_ast::MediaType;
-  use deno_ast::ParseParams;
-  use deno_ast::SourceTextInfo;
-  use deno_core::ExtensionFileSourceCode;
-  use std::path::Path;
-
-  // Always transpile `node:` built-in modules, since they might be TypeScript.
-  let media_type = if source.specifier.starts_with("node:") {
-    MediaType::TypeScript
-  } else {
-    MediaType::from_path(Path::new(&source.specifier))
-  };
-
-  match media_type {
-    MediaType::TypeScript => {}
-    MediaType::JavaScript => return Ok(()),
-    MediaType::Mjs => return Ok(()),
-    _ => panic!(
-      "Unsupported media type for snapshotting {media_type:?} for file {}",
-      source.specifier
-    ),
-  }
-  let code = source.load()?;
-
-  let parsed = deno_ast::parse_module(ParseParams {
-    specifier: source.specifier.to_string(),
-    text_info: SourceTextInfo::from_string(code.as_str().to_owned()),
-    media_type,
-    capture_tokens: false,
-    scope_analysis: false,
-    maybe_syntax: None,
-  })?;
-  let transpiled_source = parsed.transpile(&deno_ast::EmitOptions {
-    imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove,
-    inline_source_map: false,
-    ..Default::default()
-  })?;
-
-  source.code =
-    ExtensionFileSourceCode::Computed(transpiled_source.text.into());
-  Ok(())
-}
-
 /// This worker is created and used by almost all
 /// subcommands in Deno executable.
 ///
@@ -380,6 +307,7 @@ impl MainWorker {
       }
       #[cfg(feature = "__runtime_js_sources")]
       {
+        use crate::shared::maybe_transpile_source;
         for source in extension.esm_files.to_mut() {
           maybe_transpile_source(source).unwrap();
         }