feat: Standalone lite binaries and cross compilation (#9141)

This commit adds --target and --lite flags to deno compile subcommand.

--target allows to cross-compile binary to different target architectures by
fetching appropriate binary from remote server on first run. All downloaded
binaries are stored in "$DENO_DIR/dl".

--lite allows to use lite version of the runtime (ie. the one that doesn't contain
built-in tooling like formatter or linter).

cli/flags.rs

@@ -29,6 +29,8 @@ pub enum DenoSubcommand {
     source_file: String,
     output: Option<PathBuf>,
     args: Vec<String>,
+    target: Option<String>,
+    lite: bool,
   },
   Completions {
     buf: Box<[u8]>,
@@ -447,11 +449,15 @@ fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   let args = script.split_off(1);
   let source_file = script[0].to_string();
   let output = matches.value_of("output").map(PathBuf::from);
+  let lite = matches.is_present("lite");
+  let target = matches.value_of("target").map(String::from);
 
   flags.subcommand = DenoSubcommand::Compile {
     source_file,
     output,
     args,
+    lite,
+    target,
   };
 }
 
@@ -893,11 +899,24 @@ fn compile_subcommand<'a, 'b>() -> App<'a, 'b> {
         .help("Output file (defaults to $PWD/<inferred-name>)")
         .takes_value(true)
     )
+    .arg(
+      Arg::with_name("target")
+        .long("target")
+        .help("Target OS architecture")
+        .takes_value(true)
+        .possible_values(&["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "x86_64-apple-darwin"])
+    )
+    .arg(
+      Arg::with_name("lite")
+        .long("lite")
+        .help("Use lite runtime")
+    )
     .about("Compile the script into a self contained executable")
     .long_about(
       "Compiles the given script into a self contained executable.
-  deno compile --unstable https://deno.land/std/http/file_server.ts
+  deno compile --unstable -A https://deno.land/std/http/file_server.ts
   deno compile --unstable --output /usr/local/bin/color_util https://deno.land/std/examples/colors.ts
+  deno compile --unstable --lite --target x86_64-unknown-linux-gnu -A https://deno.land/std/http/file_server.ts
 
 Any flags passed which affect runtime behavior, such as '--unstable',
 '--allow-*', '--v8-flags', etc. are encoded into the output executable and used
@@ -910,8 +929,13 @@ The executable name is inferred by default:
     and the path has no parent, take the file name of the parent path. Otherwise
     settle with the generic name.
   - If the resulting name has an '@...' suffix, strip it.
+  
+This commands supports cross-compiling to different target architectures using `--target` flag.
+On the first invocation with deno will download proper binary and cache it in $DENO_DIR.
 
-Cross compiling binaries for different platforms is not currently possible.",
+It is possible to use \"lite\" binaries when compiling by passing `--lite` flag; these are stripped down versions
+of the deno binary that do not contain built-in tooling (eg. formatter, linter). This feature is experimental.
+",
     )
 }
 
@@ -3318,6 +3342,7 @@ mod tests {
     let r = flags_from_vec(svec![
       "deno",
       "compile",
+      "--lite",
       "https://deno.land/std/examples/colors.ts"
     ]);
     assert_eq!(
@@ -3327,6 +3352,8 @@ mod tests {
           source_file: "https://deno.land/std/examples/colors.ts".to_string(),
           output: None,
           args: vec![],
+          target: None,
+          lite: true,
         },
         ..Flags::default()
       }
@@ -3344,6 +3371,8 @@ mod tests {
           source_file: "https://deno.land/std/examples/colors.ts".to_string(),
           output: Some(PathBuf::from("colors")),
           args: svec!["foo", "bar"],
+          target: None,
+          lite: false,
         },
         unstable: true,
         import_map_path: Some("import_map.json".to_string()),

cli/main.rs

@@ -299,6 +299,8 @@ async fn compile_command(
   source_file: String,
   output: Option<PathBuf>,
   args: Vec<String>,
+  target: Option<String>,
+  lite: bool,
 ) -> Result<(), AnyError> {
   if !flags.unstable {
     exit_unstable("compile");
@@ -311,6 +313,7 @@ async fn compile_command(
 
   let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?;
   let program_state = ProgramState::new(flags.clone())?;
+  let deno_dir = &program_state.dir;
 
   let output = output.or_else(|| {
     infer_name_from_url(module_specifier.as_url()).map(PathBuf::from)
@@ -337,15 +340,21 @@ async fn compile_command(
     colors::green("Compile"),
     module_specifier.to_string()
   );
-  tools::standalone::create_standalone_binary(
+
+  // Select base binary based on `target` and `lite` arguments
+  let original_binary =
+    tools::standalone::get_base_binary(deno_dir, target, lite).await?;
+
+  let final_bin = tools::standalone::create_standalone_binary(
+    original_binary,
     bundle_str,
     run_flags,
-    output.clone(),
-  )
-  .await?;
+  )?;
 
   info!("{} {}", colors::green("Emit"), output.display());
 
+  tools::standalone::write_standalone_binary(output.clone(), final_bin).await?;
+
   Ok(())
 }
 
@@ -1162,7 +1171,10 @@ fn get_subcommand(
       source_file,
       output,
       args,
-    } => compile_command(flags, source_file, output, args).boxed_local(),
+      lite,
+      target,
+    } => compile_command(flags, source_file, output, args, target, lite)
+      .boxed_local(),
     DenoSubcommand::Fmt {
       check,
       files,

cli/tools/standalone.rs

@@ -1,28 +1,93 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
+use crate::deno_dir::DenoDir;
 use crate::flags::DenoSubcommand;
 use crate::flags::Flags;
 use deno_core::error::bail;
 use deno_core::error::AnyError;
 use deno_core::serde_json;
+use deno_runtime::deno_fetch::reqwest::Client;
+use std::env;
 use std::fs::read;
 use std::fs::File;
 use std::io::Read;
 use std::io::Seek;
 use std::io::SeekFrom;
 use std::io::Write;
+use std::path::Path;
 use std::path::PathBuf;
 
 use crate::standalone::Metadata;
 use crate::standalone::MAGIC_TRAILER;
 
+pub async fn get_base_binary(
+  deno_dir: &DenoDir,
+  target: Option<String>,
+  lite: bool,
+) -> Result<Vec<u8>, AnyError> {
+  if target.is_none() && !lite {
+    let path = std::env::current_exe()?;
+    return Ok(tokio::fs::read(path).await?);
+  }
+
+  let target = target.unwrap_or_else(|| env!("TARGET").to_string());
+  let exe_name = if lite { "denort" } else { "deno" };
+  let binary_name = format!("{}-{}.zip", exe_name, target);
+
+  let binary_path_suffix = if crate::version::is_canary() {
+    format!("canary/{}/{}", crate::version::GIT_COMMIT_HASH, binary_name)
+  } else {
+    format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name)
+  };
+
+  let download_directory = deno_dir.root.join("dl");
+  let binary_path = download_directory.join(&binary_path_suffix);
+
+  if !binary_path.exists() {
+    download_base_binary(&download_directory, &binary_path_suffix).await?;
+  }
+
+  let archive_data = tokio::fs::read(binary_path).await?;
+  let base_binary_path = crate::tools::upgrade::unpack(archive_data, exe_name)?;
+  let base_binary = tokio::fs::read(base_binary_path).await?;
+  Ok(base_binary)
+}
+
+async fn download_base_binary(
+  output_directory: &Path,
+  binary_path_suffix: &str,
+) -> Result<(), AnyError> {
+  let download_url = format!("https://dl.deno.land/{}", binary_path_suffix);
+
+  let client_builder = Client::builder();
+  let client = client_builder.build()?;
+
+  println!("Checking {}", &download_url);
+
+  let res = client.get(&download_url).send().await?;
+
+  let binary_content = if res.status().is_success() {
+    println!("Download has been found");
+    res.bytes().await?.to_vec()
+  } else {
+    println!("Download could not be found, aborting");
+    std::process::exit(1)
+  };
+
+  std::fs::create_dir_all(&output_directory)?;
+  let output_path = output_directory.join(binary_path_suffix);
+  std::fs::create_dir_all(&output_path.parent().unwrap())?;
+  tokio::fs::write(output_path, binary_content).await?;
+  Ok(())
+}
+
 /// This functions creates a standalone deno binary by appending a bundle
 /// and magic trailer to the currently executing binary.
-pub async fn create_standalone_binary(
+pub fn create_standalone_binary(
+  mut original_bin: Vec<u8>,
   source_code: String,
   flags: Flags,
-  output: PathBuf,
-) -> Result<(), AnyError> {
+) -> Result<Vec<u8>, AnyError> {
   let mut source_code = source_code.as_bytes().to_vec();
   let ca_data = match &flags.ca_file {
     Some(ca_file) => Some(read(ca_file)?),
@@ -39,8 +104,6 @@ pub async fn create_standalone_binary(
     ca_data,
   };
   let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec();
-  let original_binary_path = std::env::current_exe()?;
-  let mut original_bin = tokio::fs::read(original_binary_path).await?;
 
   let bundle_pos = original_bin.len();
   let metadata_pos = bundle_pos + source_code.len();
@@ -55,6 +118,15 @@ pub async fn create_standalone_binary(
   final_bin.append(&mut metadata);
   final_bin.append(&mut trailer);
 
+  Ok(final_bin)
+}
+
+/// This function writes out a final binary to specified path. If output path
+/// is not already standalone binary it will return error instead.
+pub async fn write_standalone_binary(
+  output: PathBuf,
+  final_bin: Vec<u8>,
+) -> Result<(), AnyError> {
   let output =
     if cfg!(windows) && output.extension().unwrap_or_default() != "exe" {
       PathBuf::from(output.display().to_string() + ".exe")

cli/tools/upgrade.rs

@@ -111,7 +111,7 @@ pub async fn upgrade_command(
   println!("Deno is upgrading to version {}", &install_version);
 
   let old_exe_path = std::env::current_exe()?;
-  let new_exe_path = unpack(archive_data)?;
+  let new_exe_path = unpack(archive_data, "deno")?;
   let permissions = fs::metadata(&old_exe_path)?.permissions();
   fs::set_permissions(&new_exe_path, permissions)?;
   check_exe(&new_exe_path)?;
@@ -176,13 +176,17 @@ async fn download_package(
   }
 }
 
-fn unpack(archive_data: Vec<u8>) -> Result<PathBuf, std::io::Error> {
+pub fn unpack(
+  archive_data: Vec<u8>,
+  exe_name: &str,
+) -> Result<PathBuf, std::io::Error> {
   // We use into_path so that the tempdir is not automatically deleted. This is
   // useful for debugging upgrade, but also so this function can return a path
   // to the newly uncompressed file without fear of the tempdir being deleted.
   let temp_dir = TempDir::new()?.into_path();
   let exe_ext = if cfg!(windows) { "exe" } else { "" };
-  let exe_path = temp_dir.join("deno").with_extension(exe_ext);
+  let archive_path = temp_dir.join(exe_name).with_extension(".zip");
+  let exe_path = temp_dir.join(exe_name).with_extension(exe_ext);
   assert!(!exe_path.exists());
 
   let archive_ext = Path::new(&*ARCHIVE_NAME)
@@ -191,7 +195,6 @@ fn unpack(archive_data: Vec<u8>) -> Result<PathBuf, std::io::Error> {
     .unwrap();
   let unpack_status = match archive_ext {
     "zip" if cfg!(windows) => {
-      let archive_path = temp_dir.join("deno.zip");
       fs::write(&archive_path, &archive_data)?;
       Command::new("powershell.exe")
         .arg("-NoLogo")
@@ -217,7 +220,6 @@ fn unpack(archive_data: Vec<u8>) -> Result<PathBuf, std::io::Error> {
         .wait()?
     }
     "zip" => {
-      let archive_path = temp_dir.join("deno.zip");
       fs::write(&archive_path, &archive_data)?;
       Command::new("unzip")
         .current_dir(&temp_dir)