From 7afa3aceb04e6b2c8820b7326d6f648db6b571c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 4 Mar 2023 20:39:48 -0400 Subject: [PATCH] refactor(runtime): factor out deno_io extension crate (#18001) This is a prerequisite to factor out FS ops to a separate crate. --- Cargo.lock | 12 +++++ Cargo.toml | 2 + cli/build.rs | 1 + cli/lsp/testing/execution.rs | 4 +- cli/tools/test.rs | 4 +- cli/util/console.rs | 2 +- cli/worker.rs | 6 +-- {runtime/js => ext/io}/12_io.js | 0 ext/io/Cargo.toml | 25 +++++++++++ ext/io/README.md | 4 ++ runtime/ops/io.rs => ext/io/lib.rs | 72 +++++++++++++++--------------- runtime/Cargo.toml | 2 + runtime/build.rs | 3 +- runtime/js/40_files.js | 2 +- runtime/js/40_process.js | 2 +- runtime/js/90_deno_ns.js | 2 +- runtime/lib.rs | 1 + runtime/ops/fs.rs | 2 +- runtime/ops/mod.rs | 1 - runtime/ops/process.rs | 8 ++-- runtime/ops/spawn.rs | 6 +-- runtime/ops/tty.rs | 2 +- runtime/web_worker.rs | 5 +-- runtime/worker.rs | 5 +-- 24 files changed, 108 insertions(+), 65 deletions(-) rename {runtime/js => ext/io}/12_io.js (100%) create mode 100644 ext/io/Cargo.toml create mode 100644 ext/io/README.md rename runtime/ops/io.rs => ext/io/lib.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 180f4afb88..9c6c30f3de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,6 +1161,17 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "deno_io" +version = "0.1.0" +dependencies = [ + "deno_core", + "nix", + "once_cell", + "tokio", + "winapi", +] + [[package]] name = "deno_lint" version = "0.41.0" @@ -1265,6 +1276,7 @@ dependencies = [ "deno_ffi", "deno_flash", "deno_http", + "deno_io", "deno_napi", "deno_net", "deno_node", diff --git a/Cargo.toml b/Cargo.toml index b4131c6364..94b8ea470a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "ext/flash", "ext/ffi", "ext/http", + "ext/io", "ext/net", "ext/node", "ext/url", @@ -62,6 +63,7 @@ deno_fetch = { version = "0.115.0", path = "./ext/fetch" } deno_ffi = { version = "0.78.0", path = "./ext/ffi" } deno_flash = { version = "0.27.0", path = "./ext/flash" } deno_http = { version = "0.86.0", path = "./ext/http" } +deno_io = { version = "0.1.0", path = "./ext/io" } deno_net = { version = "0.83.0", path = "./ext/net" } deno_node = { version = "0.28.0", path = "./ext/node" } deno_tls = { version = "0.78.0", path = "./ext/tls" } diff --git a/cli/build.rs b/cli/build.rs index 01127fded2..c737c1f6d1 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -351,6 +351,7 @@ fn create_cli_snapshot(snapshot_path: PathBuf) { ), deno_napi::init::(), deno_http::init(), + deno_io::init(Default::default()), deno_flash::init::(false), // No --unstable ]; diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index 28f15a7ce4..af890cc0da 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -27,8 +27,8 @@ use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::RwLock; use deno_core::ModuleSpecifier; -use deno_runtime::ops::io::Stdio; -use deno_runtime::ops::io::StdioPipe; +use deno_runtime::deno_io::Stdio; +use deno_runtime::deno_io::StdioPipe; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::tokio_util::run_local; diff --git a/cli/tools/test.rs b/cli/tools/test.rs index ae831b95be..0a0f186fb4 100644 --- a/cli/tools/test.rs +++ b/cli/tools/test.rs @@ -31,9 +31,9 @@ use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_runtime::deno_io::Stdio; +use deno_runtime::deno_io::StdioPipe; use deno_runtime::fmt_errors::format_js_error; -use deno_runtime::ops::io::Stdio; -use deno_runtime::ops::io::StdioPipe; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::tokio_util::run_local; diff --git a/cli/util/console.rs b/cli/util/console.rs index ac78ca4df5..dc37138b1b 100644 --- a/cli/util/console.rs +++ b/cli/util/console.rs @@ -4,6 +4,6 @@ use deno_runtime::ops::tty::ConsoleSize; /// Gets the console size. pub fn console_size() -> Option { - let stderr = &deno_runtime::ops::io::STDERR_HANDLE; + let stderr = &deno_runtime::deno_io::STDERR_HANDLE; deno_runtime::ops::tty::console_size(stderr).ok() } diff --git a/cli/worker.rs b/cli/worker.rs index a112663c9b..7ef90d79fd 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -420,7 +420,7 @@ pub async fn create_main_worker_for_test_or_bench( main_module: ModuleSpecifier, permissions: PermissionsContainer, custom_extensions: Vec, - stdio: deno_runtime::ops::io::Stdio, + stdio: deno_runtime::deno_io::Stdio, ) -> Result { create_main_worker_internal( ps, @@ -438,7 +438,7 @@ async fn create_main_worker_internal( main_module: ModuleSpecifier, permissions: PermissionsContainer, mut custom_extensions: Vec, - stdio: deno_runtime::ops::io::Stdio, + stdio: deno_runtime::deno_io::Stdio, bench_or_test: bool, ) -> Result { let (main_module, is_main_cjs) = if let Ok(package_ref) = @@ -642,7 +642,7 @@ fn create_web_worker_pre_execute_module_callback( fn create_web_worker_callback( ps: ProcState, - stdio: deno_runtime::ops::io::Stdio, + stdio: deno_runtime::deno_io::Stdio, ) -> Arc { Arc::new(move |args| { let maybe_inspector_server = ps.maybe_inspector_server.clone(); diff --git a/runtime/js/12_io.js b/ext/io/12_io.js similarity index 100% rename from runtime/js/12_io.js rename to ext/io/12_io.js diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml new file mode 100644 index 0000000000..7ade84c987 --- /dev/null +++ b/ext/io/Cargo.toml @@ -0,0 +1,25 @@ +# Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_io" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "IO promitives for Deno extensions" + +[lib] +path = "lib.rs" + +[dependencies] +deno_core.workspace = true +once_cell.workspace = true +tokio.workspace = true + +[target.'cfg(unix)'.dependencies] +nix.workspace = true + +[target.'cfg(windows)'.dependencies] +winapi = { workspace = true, features = ["winbase"] } diff --git a/ext/io/README.md b/ext/io/README.md new file mode 100644 index 0000000000..b66dda76e6 --- /dev/null +++ b/ext/io/README.md @@ -0,0 +1,4 @@ +# deno_io + +This crate provides IO primitives for other Deno extensions, this includes stdio +streams and abstraction over File System files. diff --git a/runtime/ops/io.rs b/ext/io/lib.rs similarity index 99% rename from runtime/ops/io.rs rename to ext/io/lib.rs index d0ee116a5b..f8dde139f3 100644 --- a/runtime/ops/io.rs +++ b/ext/io/lib.rs @@ -2,6 +2,7 @@ use deno_core::error::resource_unavailable; use deno_core::error::AnyError; +use deno_core::include_js_files; use deno_core::op; use deno_core::parking_lot::Mutex; use deno_core::AsyncMutFuture; @@ -77,46 +78,14 @@ pub static STDERR_HANDLE: Lazy = Lazy::new(|| { unsafe { StdFile::from_raw_handle(GetStdHandle(winbase::STD_ERROR_HANDLE)) } }); -pub fn init() -> Extension { - Extension::builder("deno_io") - .ops(vec![op_read_sync::decl(), op_write_sync::decl()]) - .build() -} - -pub enum StdioPipe { - Inherit, - File(StdFile), -} - -impl Default for StdioPipe { - fn default() -> Self { - Self::Inherit - } -} - -impl Clone for StdioPipe { - fn clone(&self) -> Self { - match self { - StdioPipe::Inherit => StdioPipe::Inherit, - StdioPipe::File(pipe) => StdioPipe::File(pipe.try_clone().unwrap()), - } - } -} - -/// Specify how stdin, stdout, and stderr are piped. -/// By default, inherits from the process. -#[derive(Clone, Default)] -pub struct Stdio { - pub stdin: StdioPipe, - pub stdout: StdioPipe, - pub stderr: StdioPipe, -} - -pub fn init_stdio(stdio: Stdio) -> Extension { +pub fn init(stdio: Stdio) -> Extension { // todo(dsheret): don't do this? Taking out the writers was necessary to prevent invalid handle panics let stdio = Rc::new(RefCell::new(Some(stdio))); - Extension::builder("deno_stdio") + Extension::builder("deno_io") + .ops(vec![op_read_sync::decl(), op_write_sync::decl()]) + .dependencies(vec!["deno_web"]) + .esm(include_js_files!("12_io.js",)) .middleware(|op| match op.name { "op_print" => op_print::decl(), _ => op, @@ -168,6 +137,35 @@ pub fn init_stdio(stdio: Stdio) -> Extension { .build() } +pub enum StdioPipe { + Inherit, + File(StdFile), +} + +impl Default for StdioPipe { + fn default() -> Self { + Self::Inherit + } +} + +impl Clone for StdioPipe { + fn clone(&self) -> Self { + match self { + StdioPipe::Inherit => StdioPipe::Inherit, + StdioPipe::File(pipe) => StdioPipe::File(pipe.try_clone().unwrap()), + } + } +} + +/// Specify how stdin, stdout, and stderr are piped. +/// By default, inherits from the process. +#[derive(Clone, Default)] +pub struct Stdio { + pub stdin: StdioPipe, + pub stdout: StdioPipe, + pub stderr: StdioPipe, +} + #[cfg(unix)] use nix::sys::termios; diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 7fb759b0f5..5a4028349a 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -43,6 +43,7 @@ deno_fetch.workspace = true deno_ffi.workspace = true deno_flash.workspace = true deno_http.workspace = true +deno_io.workspace = true deno_net.workspace = true deno_node.workspace = true deno_tls.workspace = true @@ -70,6 +71,7 @@ deno_fetch.workspace = true deno_ffi.workspace = true deno_flash.workspace = true deno_http.workspace = true +deno_io.workspace = true deno_napi.workspace = true deno_net.workspace = true deno_node.workspace = true diff --git a/runtime/build.rs b/runtime/build.rs index 4678eab997..fd93b90019 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -190,6 +190,7 @@ mod startup_snapshot { "deno_napi", "deno_http", "deno_flash", + "deno_io", ]) .esm(include_js_files!( dir "js", @@ -199,7 +200,6 @@ mod startup_snapshot { "06_util.js", "10_permissions.js", "11_workers.js", - "12_io.js", "13_buffer.js", "30_fs.js", "30_os.js", @@ -245,6 +245,7 @@ mod startup_snapshot { ), deno_napi::init::(), deno_http::init(), + deno_io::init(Default::default()), deno_flash::init::(false), // No --unstable runtime_extension, // FIXME(bartlomieju): these extensions are specified last, because they diff --git a/runtime/js/40_files.js b/runtime/js/40_files.js index f380ca7ebe..6548952545 100644 --- a/runtime/js/40_files.js +++ b/runtime/js/40_files.js @@ -2,7 +2,7 @@ const core = globalThis.Deno.core; const ops = core.ops; -import { read, readSync, write, writeSync } from "internal:runtime/js/12_io.js"; +import { read, readSync, write, writeSync } from "internal:deno_io/12_io.js"; import { fstat, fstatSync, diff --git a/runtime/js/40_process.js b/runtime/js/40_process.js index a949e48ed6..601c799753 100644 --- a/runtime/js/40_process.js +++ b/runtime/js/40_process.js @@ -3,7 +3,7 @@ const core = globalThis.Deno.core; const ops = core.ops; import { FsFile } from "internal:runtime/js/40_files.js"; -import { readAll } from "internal:runtime/js/12_io.js"; +import { readAll } from "internal:deno_io/12_io.js"; import { pathFromURL } from "internal:runtime/js/06_util.js"; import { assert } from "internal:deno_web/00_infra.js"; const primordials = globalThis.__bootstrap.primordials; diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index 7b9d8e6e60..8ad285e423 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -14,7 +14,7 @@ import * as build from "internal:runtime/js/01_build.js"; import * as errors from "internal:runtime/js/01_errors.js"; import * as version from "internal:runtime/js/01_version.ts"; import * as permissions from "internal:runtime/js/10_permissions.js"; -import * as io from "internal:runtime/js/12_io.js"; +import * as io from "internal:deno_io/12_io.js"; import * as buffer from "internal:runtime/js/13_buffer.js"; import * as fs from "internal:runtime/js/30_fs.js"; import * as os from "internal:runtime/js/30_os.js"; diff --git a/runtime/lib.rs b/runtime/lib.rs index 26e8d9bb4a..6bb84698d1 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -9,6 +9,7 @@ pub use deno_fetch; pub use deno_ffi; pub use deno_flash; pub use deno_http; +pub use deno_io; pub use deno_napi; pub use deno_net; pub use deno_node; diff --git a/runtime/ops/fs.rs b/runtime/ops/fs.rs index 3b22bb696d..546d1d68a7 100644 --- a/runtime/ops/fs.rs +++ b/runtime/ops/fs.rs @@ -1,6 +1,5 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. // Some deserializer fields are only used on Unix and Windows build fails without it -use super::io::StdFileResource; use super::utils::into_string; use crate::fs_util::canonicalize_path; use deno_core::error::custom_error; @@ -15,6 +14,7 @@ use deno_core::ResourceId; use deno_core::ZeroCopyBuf; use deno_crypto::rand::thread_rng; use deno_crypto::rand::Rng; +use deno_io::StdFileResource; use log::debug; use serde::Deserialize; use serde::Serialize; diff --git a/runtime/ops/mod.rs b/runtime/ops/mod.rs index ce7c52d64b..0564474b1f 100644 --- a/runtime/ops/mod.rs +++ b/runtime/ops/mod.rs @@ -3,7 +3,6 @@ pub mod fs; pub mod fs_events; pub mod http; -pub mod io; pub mod os; pub mod permissions; pub mod process; diff --git a/runtime/ops/process.rs b/runtime/ops/process.rs index 9c7a3243a3..ca37c08b74 100644 --- a/runtime/ops/process.rs +++ b/runtime/ops/process.rs @@ -1,12 +1,12 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use super::io::ChildStderrResource; -use super::io::ChildStdinResource; -use super::io::ChildStdoutResource; -use super::io::StdFileResource; use crate::permissions::PermissionsContainer; use deno_core::error::AnyError; use deno_core::op; +use deno_io::ChildStderrResource; +use deno_io::ChildStdinResource; +use deno_io::ChildStdoutResource; +use deno_io::StdFileResource; use deno_core::serde_json; use deno_core::AsyncMutFuture; diff --git a/runtime/ops/spawn.rs b/runtime/ops/spawn.rs index 089a53c238..884c462296 100644 --- a/runtime/ops/spawn.rs +++ b/runtime/ops/spawn.rs @@ -1,8 +1,5 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use super::io::ChildStderrResource; -use super::io::ChildStdinResource; -use super::io::ChildStdoutResource; use super::process::Stdio; use super::process::StdioOrRid; use crate::permissions::PermissionsContainer; @@ -13,6 +10,9 @@ use deno_core::OpState; use deno_core::Resource; use deno_core::ResourceId; use deno_core::ZeroCopyBuf; +use deno_io::ChildStderrResource; +use deno_io::ChildStdinResource; +use deno_io::ChildStdoutResource; use serde::Deserialize; use serde::Serialize; use std::borrow::Cow; diff --git a/runtime/ops/tty.rs b/runtime/ops/tty.rs index 739b84ab3d..bd9331d4ff 100644 --- a/runtime/ops/tty.rs +++ b/runtime/ops/tty.rs @@ -1,10 +1,10 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use super::io::StdFileResource; use deno_core::error::AnyError; use deno_core::op; use deno_core::Extension; use deno_core::OpState; +use deno_io::StdFileResource; use std::io::Error; #[cfg(unix)] diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index d52ea337fb..fbf10905fd 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -2,7 +2,6 @@ use crate::colors; use crate::inspector_server::InspectorServer; use crate::ops; -use crate::ops::io::Stdio; use crate::permissions::PermissionsContainer; use crate::tokio_util::run_local; use crate::worker::FormatJsErrorFn; @@ -33,6 +32,7 @@ use deno_core::RuntimeOptions; use deno_core::SharedArrayBufferStore; use deno_core::Snapshot; use deno_core::SourceMapGetter; +use deno_io::Stdio; use deno_node::RequireNpmResolver; use deno_tls::rustls::RootCertStore; use deno_web::create_entangled_message_port; @@ -426,8 +426,7 @@ impl WebWorker { // Extensions providing Deno.* features ops::fs_events::init(), ops::fs::init::(), - ops::io::init(), - ops::io::init_stdio(options.stdio), + deno_io::init(options.stdio), deno_tls::init(), deno_net::init::( options.root_cert_store.clone(), diff --git a/runtime/worker.rs b/runtime/worker.rs index 1de7a86174..1f567837a8 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -28,6 +28,7 @@ use deno_core::RuntimeOptions; use deno_core::SharedArrayBufferStore; use deno_core::Snapshot; use deno_core::SourceMapGetter; +use deno_io::Stdio; use deno_node::RequireNpmResolver; use deno_tls::rustls::RootCertStore; use deno_web::BlobStore; @@ -35,7 +36,6 @@ use log::debug; use crate::inspector_server::InspectorServer; use crate::ops; -use crate::ops::io::Stdio; use crate::permissions::PermissionsContainer; use crate::BootstrapOptions; @@ -257,8 +257,7 @@ impl MainWorker { ops::spawn::init(), ops::fs_events::init(), ops::fs::init::(), - ops::io::init(), - ops::io::init_stdio(options.stdio), + deno_io::init(options.stdio), deno_tls::init(), deno_net::init::( options.root_cert_store.clone(),