feat: deno compile (#8539)

cli/flags.rs

@@ -22,6 +22,10 @@ pub enum DenoSubcommand {
     source_file: String,
     out_file: Option<PathBuf>,
   },
+  Compile {
+    source_file: String,
+    out_file: Option<String>,
+  },
   Completions {
     buf: Box<[u8]>,
   },
@@ -292,6 +296,8 @@ pub fn flags_from_vec_safe(args: Vec<String>) -> clap::Result<Flags> {
     doc_parse(&mut flags, m);
   } else if let Some(m) = matches.subcommand_matches("lint") {
     lint_parse(&mut flags, m);
+  } else if let Some(m) = matches.subcommand_matches("compile") {
+    compile_parse(&mut flags, m);
   } else {
     repl_parse(&mut flags, &matches);
   }
@@ -341,6 +347,7 @@ If the flag is set, restrict these messages to errors.",
     )
     .subcommand(bundle_subcommand())
     .subcommand(cache_subcommand())
+    .subcommand(compile_subcommand())
     .subcommand(completions_subcommand())
     .subcommand(doc_subcommand())
     .subcommand(eval_subcommand())
@@ -408,6 +415,18 @@ fn install_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   };
 }
 
+fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
+  compile_args_parse(flags, matches);
+
+  let source_file = matches.value_of("source_file").unwrap().to_string();
+  let out_file = matches.value_of("out_file").map(|s| s.to_string());
+
+  flags.subcommand = DenoSubcommand::Compile {
+    source_file,
+    out_file,
+  };
+}
+
 fn bundle_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   compile_args_parse(flags, matches);
 
@@ -802,6 +821,32 @@ The installation root is determined, in order of precedence:
 These must be added to the path manually if required.")
 }
 
+fn compile_subcommand<'a, 'b>() -> App<'a, 'b> {
+  compile_args(SubCommand::with_name("compile"))
+    .arg(
+      Arg::with_name("source_file")
+        .takes_value(true)
+        .required(true),
+    )
+    .arg(Arg::with_name("out_file").takes_value(true))
+    .about("Compile the script into a self contained executable")
+    .long_about(
+      "Compiles the given script into a self contained executable.
+  deno compile --unstable https://deno.land/std/http/file_server.ts
+  deno compile --unstable https://deno.land/std/examples/colors.ts color_util
+  
+The executable name is inferred by default:
+  - Attempt to take the file stem of the URL path. The above example would
+    become 'file_server'.
+  - If the file stem is something generic like 'main', 'mod', 'index' or 'cli',
+    and the path has no parent, take the file name of the parent path. Otherwise
+    settle with the generic name.
+  - If the resulting name has an '@...' suffix, strip it.
+
+Cross compiling binaries for different platforms is not currently possible.",
+    )
+}
+
 fn bundle_subcommand<'a, 'b>() -> App<'a, 'b> {
   compile_args(SubCommand::with_name("bundle"))
     .arg(
@@ -3200,4 +3245,48 @@ mod tests {
       }
     );
   }
+
+  #[test]
+  fn compile() {
+    let r = flags_from_vec_safe(svec![
+      "deno",
+      "compile",
+      "https://deno.land/std/examples/colors.ts"
+    ]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Compile {
+          source_file: "https://deno.land/std/examples/colors.ts".to_string(),
+          out_file: None
+        },
+        ..Flags::default()
+      }
+    );
+  }
+
+  #[test]
+  fn compile_with_flags() {
+    #[rustfmt::skip]
+    let r = flags_from_vec_safe(svec!["deno", "compile", "--unstable", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--lock-write", "--cert", "example.crt", "https://deno.land/std/examples/colors.ts", "colors"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Compile {
+          source_file: "https://deno.land/std/examples/colors.ts".to_string(),
+          out_file: Some("colors".to_string())
+        },
+        unstable: true,
+        import_map_path: Some("import_map.json".to_string()),
+        no_remote: true,
+        config_path: Some("tsconfig.json".to_string()),
+        no_check: true,
+        reload: true,
+        lock: Some(PathBuf::from("lock.json")),
+        lock_write: true,
+        ca_file: Some("example.crt".to_string()),
+        ..Flags::default()
+      }
+    );
+  }
 }

cli/main.rs

@@ -39,6 +39,7 @@ mod resolve_addr;
 mod signal;
 mod source_maps;
 mod specifier_handler;
+mod standalone;
 mod text_encoding;
 mod tokio_util;
 mod tools;
@@ -51,10 +52,16 @@ mod worker;
 use crate::file_fetcher::File;
 use crate::file_fetcher::FileFetcher;
 use crate::file_watcher::ModuleResolutionResult;
+use crate::flags::DenoSubcommand;
+use crate::flags::Flags;
+use crate::import_map::ImportMap;
 use crate::media_type::MediaType;
 use crate::permissions::Permissions;
+use crate::program_state::exit_unstable;
 use crate::program_state::ProgramState;
 use crate::specifier_handler::FetchHandler;
+use crate::standalone::create_standalone_binary;
+use crate::tools::installer::infer_name_from_url;
 use crate::worker::MainWorker;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
@@ -66,12 +73,8 @@ use deno_core::v8_set_flags;
 use deno_core::ModuleSpecifier;
 use deno_doc as doc;
 use deno_doc::parser::DocFileLoader;
-use flags::DenoSubcommand;
-use flags::Flags;
-use import_map::ImportMap;
 use log::Level;
 use log::LevelFilter;
-use program_state::exit_unstable;
 use std::cell::RefCell;
 use std::env;
 use std::io::Read;
@@ -149,6 +152,56 @@ fn get_types(unstable: bool) -> String {
   types
 }
 
+async fn compile_command(
+  flags: Flags,
+  source_file: String,
+  out_file: Option<String>,
+) -> Result<(), AnyError> {
+  if !flags.unstable {
+    exit_unstable("compile");
+  }
+
+  let debug = flags.log_level == Some(log::Level::Debug);
+
+  let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?;
+  let program_state = ProgramState::new(flags.clone())?;
+
+  let out_file =
+    out_file.or_else(|| infer_name_from_url(module_specifier.as_url()));
+  let out_file = match out_file {
+    Some(out_file) => out_file,
+    None => return Err(generic_error(
+      "An executable name was not provided. One could not be inferred from the URL. Aborting.",
+    )),
+  };
+
+  let module_graph = create_module_graph_and_maybe_check(
+    module_specifier.clone(),
+    program_state.clone(),
+    debug,
+  )
+  .await?;
+
+  info!(
+    "{} {}",
+    colors::green("Bundle"),
+    module_specifier.to_string()
+  );
+  let bundle_str = bundle_module_graph(module_graph, flags, debug)?;
+
+  info!(
+    "{} {}",
+    colors::green("Compile"),
+    module_specifier.to_string()
+  );
+  create_standalone_binary(bundle_str.as_bytes().to_vec(), out_file.clone())
+    .await?;
+
+  info!("{} {}", colors::green("Emit"), out_file);
+
+  Ok(())
+}
+
 async fn info_command(
   flags: Flags,
   maybe_specifier: Option<String>,
@@ -299,6 +352,73 @@ async fn eval_command(
   Ok(())
 }
 
+async fn create_module_graph_and_maybe_check(
+  module_specifier: ModuleSpecifier,
+  program_state: Arc<ProgramState>,
+  debug: bool,
+) -> Result<module_graph::Graph, AnyError> {
+  let handler = Rc::new(RefCell::new(FetchHandler::new(
+    &program_state,
+    // when bundling, dynamic imports are only access for their type safety,
+    // therefore we will allow the graph to access any module.
+    Permissions::allow_all(),
+  )?));
+  let mut builder = module_graph::GraphBuilder::new(
+    handler,
+    program_state.maybe_import_map.clone(),
+    program_state.lockfile.clone(),
+  );
+  builder.add(&module_specifier, false).await?;
+  let module_graph = builder.get_graph();
+
+  if !program_state.flags.no_check {
+    // TODO(@kitsonk) support bundling for workers
+    let lib = if program_state.flags.unstable {
+      module_graph::TypeLib::UnstableDenoWindow
+    } else {
+      module_graph::TypeLib::DenoWindow
+    };
+    let result_info =
+      module_graph.clone().check(module_graph::CheckOptions {
+        debug,
+        emit: false,
+        lib,
+        maybe_config_path: program_state.flags.config_path.clone(),
+        reload: program_state.flags.reload,
+      })?;
+
+    debug!("{}", result_info.stats);
+    if let Some(ignored_options) = result_info.maybe_ignored_options {
+      eprintln!("{}", ignored_options);
+    }
+    if !result_info.diagnostics.is_empty() {
+      return Err(generic_error(result_info.diagnostics.to_string()));
+    }
+  }
+
+  Ok(module_graph)
+}
+
+fn bundle_module_graph(
+  module_graph: module_graph::Graph,
+  flags: Flags,
+  debug: bool,
+) -> Result<String, AnyError> {
+  let (bundle, stats, maybe_ignored_options) =
+    module_graph.bundle(module_graph::BundleOptions {
+      debug,
+      maybe_config_path: flags.config_path,
+    })?;
+  match maybe_ignored_options {
+    Some(ignored_options) if flags.no_check => {
+      eprintln!("{}", ignored_options);
+    }
+    _ => {}
+  }
+  debug!("{}", stats);
+  Ok(bundle)
+}
+
 async fn bundle_command(
   flags: Flags,
   source_file: String,
@@ -323,44 +443,12 @@ async fn bundle_command(
         module_specifier.to_string()
       );
 
-      let handler = Rc::new(RefCell::new(FetchHandler::new(
-        &program_state,
-        // when bundling, dynamic imports are only access for their type safety,
-        // therefore we will allow the graph to access any module.
-        Permissions::allow_all(),
-      )?));
-      let mut builder = module_graph::GraphBuilder::new(
-        handler,
-        program_state.maybe_import_map.clone(),
-        program_state.lockfile.clone(),
-      );
-      builder.add(&module_specifier, false).await?;
-      let module_graph = builder.get_graph();
-
-      if !flags.no_check {
-        // TODO(@kitsonk) support bundling for workers
-        let lib = if flags.unstable {
-          module_graph::TypeLib::UnstableDenoWindow
-        } else {
-          module_graph::TypeLib::DenoWindow
-        };
-        let result_info =
-          module_graph.clone().check(module_graph::CheckOptions {
-            debug,
-            emit: false,
-            lib,
-            maybe_config_path: flags.config_path.clone(),
-            reload: flags.reload,
-          })?;
-
-        debug!("{}", result_info.stats);
-        if let Some(ignored_options) = result_info.maybe_ignored_options {
-          eprintln!("{}", ignored_options);
-        }
-        if !result_info.diagnostics.is_empty() {
-          return Err(generic_error(result_info.diagnostics.to_string()));
-        }
-      }
+      let module_graph = create_module_graph_and_maybe_check(
+        module_specifier,
+        program_state.clone(),
+        debug,
+      )
+      .await?;
 
       let mut paths_to_watch: Vec<PathBuf> = module_graph
         .get_modules()
@@ -392,19 +480,7 @@ async fn bundle_command(
     let flags = flags.clone();
     let out_file = out_file.clone();
     async move {
-      let (output, stats, maybe_ignored_options) =
-        module_graph.bundle(module_graph::BundleOptions {
-          debug,
-          maybe_config_path: flags.config_path,
-        })?;
-
-      match maybe_ignored_options {
-        Some(ignored_options) if flags.no_check => {
-          eprintln!("{}", ignored_options);
-        }
-        _ => {}
-      }
-      debug!("{}", stats);
+      let output = bundle_module_graph(module_graph, flags, debug)?;
 
       debug!(">>>>> bundle END");
 
@@ -898,6 +974,10 @@ fn get_subcommand(
     DenoSubcommand::Cache { files } => {
       cache_command(flags, files).boxed_local()
     }
+    DenoSubcommand::Compile {
+      source_file,
+      out_file,
+    } => compile_command(flags, source_file, out_file).boxed_local(),
     DenoSubcommand::Fmt {
       check,
       files,
@@ -968,8 +1048,12 @@ pub fn main() {
   colors::enable_ansi(); // For Windows 10
 
   let args: Vec<String> = env::args().collect();
-  let flags = flags::flags_from_vec(args);
+  if let Err(err) = standalone::try_run_standalone_binary(args.clone()) {
+    eprintln!("{}: {}", colors::red_bold("error"), err.to_string());
+    std::process::exit(1);
+  }
 
+  let flags = flags::flags_from_vec(args);
   if let Some(ref v8_flags) = flags.v8_flags {
     init_v8_flags(v8_flags);
   }

cli/standalone.rs

@@ -0,0 +1,159 @@
+use crate::colors;
+use crate::flags::Flags;
+use crate::permissions::Permissions;
+use crate::program_state::ProgramState;
+use crate::tokio_util;
+use crate::worker::MainWorker;
+use deno_core::error::type_error;
+use deno_core::error::AnyError;
+use deno_core::futures::FutureExt;
+use deno_core::ModuleLoader;
+use deno_core::ModuleSpecifier;
+use deno_core::OpState;
+use std::cell::RefCell;
+use std::convert::TryInto;
+use std::env::current_exe;
+use std::fs::File;
+use std::io::Read;
+use std::io::Seek;
+use std::io::SeekFrom;
+use std::io::Write;
+use std::pin::Pin;
+use std::rc::Rc;
+
+const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd";
+
+/// This function will try to run this binary as a standalone binary
+/// produced by `deno compile`. It determines if this is a stanalone
+/// binary by checking for the magic trailer string `D3N0` at EOF-12.
+/// After the magic trailer is a u64 pointer to the start of the JS
+/// file embedded in the binary. This file is read, and run. If no
+/// magic trailer is present, this function exits with Ok(()).
+pub fn try_run_standalone_binary(args: Vec<String>) -> Result<(), AnyError> {
+  let current_exe_path = current_exe()?;
+
+  let mut current_exe = File::open(current_exe_path)?;
+  let trailer_pos = current_exe.seek(SeekFrom::End(-16))?;
+  let mut trailer = [0; 16];
+  current_exe.read_exact(&mut trailer)?;
+  let (magic_trailer, bundle_pos_arr) = trailer.split_at(8);
+  if magic_trailer == MAGIC_TRAILER {
+    let bundle_pos_arr: &[u8; 8] = bundle_pos_arr.try_into()?;
+    let bundle_pos = u64::from_be_bytes(*bundle_pos_arr);
+    current_exe.seek(SeekFrom::Start(bundle_pos))?;
+
+    let bundle_len = trailer_pos - bundle_pos;
+    let mut bundle = String::new();
+    current_exe.take(bundle_len).read_to_string(&mut bundle)?;
+    // TODO: check amount of bytes read
+
+    if let Err(err) = tokio_util::run_basic(run(bundle, args)) {
+      eprintln!("{}: {}", colors::red_bold("error"), err.to_string());
+      std::process::exit(1);
+    }
+    std::process::exit(0);
+  } else {
+    Ok(())
+  }
+}
+
+const SPECIFIER: &str = "file://$deno$/bundle.js";
+
+struct EmbeddedModuleLoader(String);
+
+impl ModuleLoader for EmbeddedModuleLoader {
+  fn resolve(
+    &self,
+    _op_state: Rc<RefCell<OpState>>,
+    specifier: &str,
+    _referrer: &str,
+    _is_main: bool,
+  ) -> Result<ModuleSpecifier, AnyError> {
+    if specifier != SPECIFIER {
+      return Err(type_error(
+        "Self-contained binaries don't support module loading",
+      ));
+    }
+    Ok(ModuleSpecifier::resolve_url(specifier)?)
+  }
+
+  fn load(
+    &self,
+    _op_state: Rc<RefCell<OpState>>,
+    module_specifier: &ModuleSpecifier,
+    _maybe_referrer: Option<ModuleSpecifier>,
+    _is_dynamic: bool,
+  ) -> Pin<Box<deno_core::ModuleSourceFuture>> {
+    let module_specifier = module_specifier.clone();
+    let code = self.0.to_string();
+    async move {
+      if module_specifier.to_string() != SPECIFIER {
+        return Err(type_error(
+          "Self-contained binaries don't support module loading",
+        ));
+      }
+      Ok(deno_core::ModuleSource {
+        code,
+        module_url_specified: module_specifier.to_string(),
+        module_url_found: module_specifier.to_string(),
+      })
+    }
+    .boxed_local()
+  }
+}
+
+async fn run(source_code: String, args: Vec<String>) -> Result<(), AnyError> {
+  let mut flags = Flags::default();
+  flags.argv = args[1..].to_vec();
+  // TODO(lucacasonato): remove once you can specify this correctly through embedded metadata
+  flags.unstable = true;
+  let main_module = ModuleSpecifier::resolve_url(SPECIFIER)?;
+  let program_state = ProgramState::new(flags.clone())?;
+  let permissions = Permissions::allow_all();
+  let module_loader = Rc::new(EmbeddedModuleLoader(source_code));
+  let mut worker = MainWorker::from_options(
+    &program_state,
+    main_module.clone(),
+    permissions,
+    module_loader,
+  );
+  worker.execute_module(&main_module).await?;
+  worker.execute("window.dispatchEvent(new Event('load'))")?;
+  worker.run_event_loop().await?;
+  worker.execute("window.dispatchEvent(new Event('unload'))")?;
+  Ok(())
+}
+
+/// This functions creates a standalone deno binary by appending a bundle
+/// and magic trailer to the currently executing binary.
+pub async fn create_standalone_binary(
+  mut source_code: Vec<u8>,
+  out_file: String,
+) -> Result<(), AnyError> {
+  let original_binary_path = std::env::current_exe()?;
+  let mut original_bin = tokio::fs::read(original_binary_path).await?;
+
+  let mut trailer = MAGIC_TRAILER.to_vec();
+  trailer.write_all(&original_bin.len().to_be_bytes())?;
+
+  let mut final_bin =
+    Vec::with_capacity(original_bin.len() + source_code.len() + trailer.len());
+  final_bin.append(&mut original_bin);
+  final_bin.append(&mut source_code);
+  final_bin.append(&mut trailer);
+
+  let out_file = if cfg!(windows) && !out_file.ends_with(".exe") {
+    format!("{}.exe", out_file)
+  } else {
+    out_file
+  };
+  tokio::fs::write(&out_file, final_bin).await?;
+  #[cfg(unix)]
+  {
+    use std::os::unix::fs::PermissionsExt;
+    let perms = std::fs::Permissions::from_mode(0o777);
+    tokio::fs::set_permissions(out_file, perms).await?;
+  }
+
+  Ok(())
+}

cli/tests/integration_tests.rs

@@ -98,7 +98,6 @@ fn eval_p() {
 }
 
 #[test]
-
 fn run_from_stdin() {
   let mut deno = util::deno_cmd()
     .current_dir(util::root_path())
@@ -4529,3 +4528,100 @@ fn fmt_ignore_unexplicit_files() {
   assert!(output.status.success());
   assert_eq!(output.stderr, b"Checked 0 file\n");
 }
+
+#[test]
+fn compile() {
+  let dir = TempDir::new().expect("tempdir fail");
+  let exe = if cfg!(windows) {
+    dir.path().join("welcome.exe")
+  } else {
+    dir.path().join("welcome")
+  };
+  let output = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("compile")
+    .arg("--unstable")
+    .arg("./std/examples/welcome.ts")
+    .arg(&exe)
+    .stdout(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(output.status.success());
+  let output = Command::new(exe)
+    .stdout(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(output.status.success());
+  assert_eq!(output.stdout, "Welcome to Deno 🦕\n".as_bytes());
+}
+
+#[test]
+fn standalone_args() {
+  let dir = TempDir::new().expect("tempdir fail");
+  let exe = if cfg!(windows) {
+    dir.path().join("args.exe")
+  } else {
+    dir.path().join("args")
+  };
+  let output = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("compile")
+    .arg("--unstable")
+    .arg("./cli/tests/028_args.ts")
+    .arg(&exe)
+    .stdout(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(output.status.success());
+  let output = Command::new(exe)
+    .arg("foo")
+    .arg("--bar")
+    .arg("--unstable")
+    .stdout(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(output.status.success());
+  assert_eq!(output.stdout, b"foo\n--bar\n--unstable\n");
+}
+
+#[test]
+fn standalone_no_module_load() {
+  let dir = TempDir::new().expect("tempdir fail");
+  let exe = if cfg!(windows) {
+    dir.path().join("hello.exe")
+  } else {
+    dir.path().join("hello")
+  };
+  let output = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("compile")
+    .arg("--unstable")
+    .arg("./cli/tests/standalone_import.ts")
+    .arg(&exe)
+    .stdout(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(output.status.success());
+  let output = Command::new(exe)
+    .stdout(std::process::Stdio::piped())
+    .stderr(std::process::Stdio::piped())
+    .spawn()
+    .unwrap()
+    .wait_with_output()
+    .unwrap();
+  assert!(!output.status.success());
+  assert_eq!(output.stdout, b"start\n");
+  let stderr_str = String::from_utf8(output.stderr).unwrap();
+  assert!(util::strip_ansi_codes(&stderr_str)
+    .contains("Self-contained binaries don't support module loading"));
+}

cli/tests/standalone_import.ts

@@ -0,0 +1,2 @@
+console.log("start");
+await import("./001_hello.js");

cli/tools/installer.rs

@@ -108,7 +108,7 @@ fn get_installer_root() -> Result<PathBuf, io::Error> {
   Ok(home_path)
 }
 
-fn infer_name_from_url(url: &Url) -> Option<String> {
+pub fn infer_name_from_url(url: &Url) -> Option<String> {
   let path = PathBuf::from(url.path());
   let mut stem = match path.file_stem() {
     Some(stem) => stem.to_string_lossy().to_string(),

cli/worker.rs

@@ -17,9 +17,11 @@ use deno_core::futures::future::FutureExt;
 use deno_core::url::Url;
 use deno_core::JsRuntime;
 use deno_core::ModuleId;
+use deno_core::ModuleLoader;
 use deno_core::ModuleSpecifier;
 use deno_core::RuntimeOptions;
 use std::env;
+use std::rc::Rc;
 use std::sync::Arc;
 use std::task::Context;
 use std::task::Poll;
@@ -45,6 +47,16 @@ impl MainWorker {
   ) -> Self {
     let module_loader =
       CliModuleLoader::new(program_state.maybe_import_map.clone());
+
+    Self::from_options(program_state, main_module, permissions, module_loader)
+  }
+
+  pub fn from_options(
+    program_state: &Arc<ProgramState>,
+    main_module: ModuleSpecifier,
+    permissions: Permissions,
+    module_loader: Rc<dyn ModuleLoader>,
+  ) -> Self {
     let global_state_ = program_state.clone();
 
     let js_error_create_fn = Box::new(move |core_js_error| {