2019-03-14 23:17:52 +00:00
|
|
|
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
|
2019-06-05 20:35:38 +00:00
|
|
|
use crate::compiler::compile_async;
|
|
|
|
use crate::compiler::ModuleMetaData;
|
2019-03-14 23:17:52 +00:00
|
|
|
use crate::deno_dir;
|
2019-06-05 20:35:38 +00:00
|
|
|
use crate::errors::DenoError;
|
2019-03-20 00:55:59 +00:00
|
|
|
use crate::errors::DenoResult;
|
2019-03-14 23:17:52 +00:00
|
|
|
use crate::flags;
|
|
|
|
use crate::global_timer::GlobalTimer;
|
2019-06-09 13:08:20 +00:00
|
|
|
use crate::import_map::ImportMap;
|
2019-06-05 20:35:38 +00:00
|
|
|
use crate::msg;
|
2019-04-09 17:11:25 +00:00
|
|
|
use crate::ops;
|
2019-03-20 00:55:59 +00:00
|
|
|
use crate::permissions::DenoPermissions;
|
2019-05-11 14:23:19 +00:00
|
|
|
use crate::progress::Progress;
|
2019-04-08 21:10:00 +00:00
|
|
|
use crate::resources;
|
2019-04-01 19:09:59 +00:00
|
|
|
use crate::resources::ResourceId;
|
2019-06-05 20:35:38 +00:00
|
|
|
use crate::tokio_util;
|
|
|
|
use crate::worker::resolve_module_spec;
|
2019-04-08 21:10:00 +00:00
|
|
|
use crate::worker::Worker;
|
2019-03-30 23:30:40 +00:00
|
|
|
use deno::Buf;
|
2019-06-05 20:35:38 +00:00
|
|
|
use deno::Loader;
|
2019-04-09 17:11:25 +00:00
|
|
|
use deno::Op;
|
2019-04-28 19:31:10 +00:00
|
|
|
use deno::PinnedBuf;
|
2019-06-05 20:35:38 +00:00
|
|
|
use futures::future::Either;
|
2019-04-01 19:09:59 +00:00
|
|
|
use futures::future::Shared;
|
2019-06-05 20:35:38 +00:00
|
|
|
use futures::Future;
|
2019-03-14 23:17:52 +00:00
|
|
|
use std;
|
2019-04-01 19:09:59 +00:00
|
|
|
use std::collections::HashMap;
|
2019-05-20 16:06:57 +00:00
|
|
|
use std::collections::HashSet;
|
2019-03-14 23:17:52 +00:00
|
|
|
use std::env;
|
2019-04-29 14:58:31 +00:00
|
|
|
use std::fs;
|
2019-04-09 17:11:25 +00:00
|
|
|
use std::ops::Deref;
|
2019-03-14 23:17:52 +00:00
|
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
2019-04-09 17:11:25 +00:00
|
|
|
use std::sync::Arc;
|
2019-03-14 23:17:52 +00:00
|
|
|
use std::sync::Mutex;
|
2019-04-08 20:22:40 +00:00
|
|
|
use std::time::Instant;
|
2019-04-08 21:10:00 +00:00
|
|
|
use tokio::sync::mpsc as async_mpsc;
|
2019-03-14 23:17:52 +00:00
|
|
|
|
|
|
|
pub type WorkerSender = async_mpsc::Sender<Buf>;
|
|
|
|
pub type WorkerReceiver = async_mpsc::Receiver<Buf>;
|
|
|
|
pub type WorkerChannels = (WorkerSender, WorkerReceiver);
|
2019-04-08 21:10:00 +00:00
|
|
|
pub type UserWorkerTable = HashMap<ResourceId, Shared<Worker>>;
|
2019-03-14 23:17:52 +00:00
|
|
|
|
|
|
|
#[derive(Default)]
|
|
|
|
pub struct Metrics {
|
|
|
|
pub ops_dispatched: AtomicUsize,
|
|
|
|
pub ops_completed: AtomicUsize,
|
|
|
|
pub bytes_sent_control: AtomicUsize,
|
|
|
|
pub bytes_sent_data: AtomicUsize,
|
|
|
|
pub bytes_received: AtomicUsize,
|
|
|
|
pub resolve_count: AtomicUsize,
|
2019-05-20 16:06:57 +00:00
|
|
|
pub compiler_starts: AtomicUsize,
|
2019-03-14 23:17:52 +00:00
|
|
|
}
|
|
|
|
|
2019-05-11 14:23:19 +00:00
|
|
|
/// Isolate cannot be passed between threads but ThreadSafeState can.
|
|
|
|
/// ThreadSafeState satisfies Send and Sync. So any state that needs to be
|
|
|
|
/// accessed outside the main V8 thread should be inside ThreadSafeState.
|
2019-04-09 17:11:25 +00:00
|
|
|
pub struct ThreadSafeState(Arc<State>);
|
|
|
|
|
2019-03-14 23:17:52 +00:00
|
|
|
#[cfg_attr(feature = "cargo-clippy", allow(stutter))]
|
2019-04-09 17:11:25 +00:00
|
|
|
pub struct State {
|
2019-06-09 13:08:20 +00:00
|
|
|
pub main_module: Option<String>,
|
2019-03-14 23:17:52 +00:00
|
|
|
pub dir: deno_dir::DenoDir,
|
|
|
|
pub argv: Vec<String>,
|
2019-03-20 00:55:59 +00:00
|
|
|
pub permissions: DenoPermissions,
|
2019-03-14 23:17:52 +00:00
|
|
|
pub flags: flags::DenoFlags,
|
2019-04-29 14:58:31 +00:00
|
|
|
/// When flags contains a `.config_path` option, the content of the
|
|
|
|
/// configuration file will be resolved and set.
|
|
|
|
pub config: Option<Vec<u8>>,
|
|
|
|
/// When flags contains a `.config_path` option, the fully qualified path
|
|
|
|
/// name of the passed path will be resolved and set.
|
|
|
|
pub config_path: Option<String>,
|
2019-06-09 13:08:20 +00:00
|
|
|
/// When flags contains a `.import_map_path` option, the content of the
|
|
|
|
/// import map file will be resolved and set.
|
|
|
|
pub import_map: Option<ImportMap>,
|
2019-03-14 23:17:52 +00:00
|
|
|
pub metrics: Metrics,
|
2019-04-08 21:10:00 +00:00
|
|
|
pub worker_channels: Mutex<WorkerChannels>,
|
2019-03-14 23:17:52 +00:00
|
|
|
pub global_timer: Mutex<GlobalTimer>,
|
2019-04-01 19:09:59 +00:00
|
|
|
pub workers: Mutex<UserWorkerTable>,
|
2019-04-08 20:22:40 +00:00
|
|
|
pub start_time: Instant,
|
2019-05-11 14:23:19 +00:00
|
|
|
/// A reference to this worker's resource.
|
2019-04-08 21:10:00 +00:00
|
|
|
pub resource: resources::Resource,
|
2019-04-11 14:58:31 +00:00
|
|
|
pub dispatch_selector: ops::OpSelector,
|
2019-05-11 14:23:19 +00:00
|
|
|
/// Reference to global progress bar.
|
|
|
|
pub progress: Progress,
|
2019-05-20 16:06:57 +00:00
|
|
|
|
|
|
|
/// Set of all URLs that have been compiled. This is a hacky way to work
|
|
|
|
/// around the fact that --reload will force multiple compilations of the same
|
|
|
|
/// module.
|
|
|
|
compiled: Mutex<HashSet<String>>,
|
2019-03-14 23:17:52 +00:00
|
|
|
}
|
|
|
|
|
2019-04-09 17:11:25 +00:00
|
|
|
impl Clone for ThreadSafeState {
|
|
|
|
fn clone(&self) -> Self {
|
|
|
|
ThreadSafeState(self.0.clone())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Deref for ThreadSafeState {
|
|
|
|
type Target = Arc<State>;
|
|
|
|
fn deref(&self) -> &Self::Target {
|
|
|
|
&self.0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-23 22:58:00 +00:00
|
|
|
impl ThreadSafeState {
|
2019-05-01 22:22:32 +00:00
|
|
|
pub fn dispatch(&self, control: &[u8], zero_copy: Option<PinnedBuf>) -> Op {
|
2019-04-11 14:58:31 +00:00
|
|
|
ops::dispatch_all(self, control, zero_copy, self.dispatch_selector)
|
2019-04-09 17:11:25 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-08 11:10:12 +00:00
|
|
|
pub fn fetch_module_meta_data_and_maybe_compile_async(
|
2019-06-05 20:35:38 +00:00
|
|
|
state: &ThreadSafeState,
|
|
|
|
specifier: &str,
|
|
|
|
referrer: &str,
|
|
|
|
) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
|
|
|
|
let state_ = state.clone();
|
|
|
|
let specifier = specifier.to_string();
|
|
|
|
let referrer = referrer.to_string();
|
2019-06-09 13:08:20 +00:00
|
|
|
let is_root = referrer == ".";
|
2019-06-05 20:35:38 +00:00
|
|
|
|
|
|
|
let f =
|
2019-06-09 13:08:20 +00:00
|
|
|
futures::future::result(state.resolve(&specifier, &referrer, is_root));
|
2019-06-05 20:35:38 +00:00
|
|
|
f.and_then(move |module_id| {
|
|
|
|
let use_cache = !state_.flags.reload || state_.has_compiled(&module_id);
|
|
|
|
let no_fetch = state_.flags.no_fetch;
|
|
|
|
|
|
|
|
state_
|
|
|
|
.dir
|
|
|
|
.fetch_module_meta_data_async(&specifier, &referrer, use_cache, no_fetch)
|
|
|
|
.and_then(move |out| {
|
|
|
|
if out.media_type == msg::MediaType::TypeScript
|
|
|
|
&& !out.has_output_code_and_source_map()
|
|
|
|
{
|
|
|
|
debug!(">>>>> compile_sync START");
|
|
|
|
Either::A(
|
2019-06-07 00:37:26 +00:00
|
|
|
compile_async(state_.clone(), &out)
|
2019-06-05 20:35:38 +00:00
|
|
|
.map_err(|e| {
|
|
|
|
debug!("compiler error exiting!");
|
|
|
|
eprintln!("\n{}", e.to_string());
|
|
|
|
std::process::exit(1);
|
|
|
|
}).and_then(move |out| {
|
|
|
|
debug!(">>>>> compile_sync END");
|
|
|
|
Ok(out)
|
|
|
|
}),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
Either::B(futures::future::ok(out))
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn fetch_module_meta_data_and_maybe_compile(
|
|
|
|
state: &ThreadSafeState,
|
|
|
|
specifier: &str,
|
|
|
|
referrer: &str,
|
|
|
|
) -> Result<ModuleMetaData, DenoError> {
|
|
|
|
tokio_util::block_on(fetch_module_meta_data_and_maybe_compile_async(
|
|
|
|
state, specifier, referrer,
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Loader for ThreadSafeState {
|
|
|
|
type Error = DenoError;
|
|
|
|
|
2019-06-09 13:08:20 +00:00
|
|
|
fn resolve(
|
|
|
|
&self,
|
|
|
|
specifier: &str,
|
|
|
|
referrer: &str,
|
|
|
|
is_root: bool,
|
|
|
|
) -> Result<String, Self::Error> {
|
|
|
|
if !is_root {
|
|
|
|
if let Some(import_map) = &self.import_map {
|
|
|
|
match import_map.resolve(specifier, referrer) {
|
|
|
|
Ok(result) => {
|
|
|
|
if result.is_some() {
|
|
|
|
return Ok(result.unwrap());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Err(err) => {
|
|
|
|
// TODO(bartlomieju): this should be coerced to DenoError
|
|
|
|
panic!("error resolving using import map: {:?}", err);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-05 20:35:38 +00:00
|
|
|
resolve_module_spec(specifier, referrer).map_err(DenoError::from)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Given an absolute url, load its source code.
|
|
|
|
fn load(&self, url: &str) -> Box<deno::SourceCodeInfoFuture<Self::Error>> {
|
|
|
|
self.metrics.resolve_count.fetch_add(1, Ordering::SeqCst);
|
|
|
|
Box::new(
|
|
|
|
fetch_module_meta_data_and_maybe_compile_async(self, url, ".")
|
|
|
|
.map_err(|err| {
|
|
|
|
eprintln!("{}", err);
|
|
|
|
err
|
|
|
|
}).map(|module_meta_data| deno::SourceCodeInfo {
|
|
|
|
// Real module name, might be different from initial URL
|
|
|
|
// due to redirections.
|
|
|
|
code: module_meta_data.js_source(),
|
|
|
|
module_name: module_meta_data.module_name,
|
|
|
|
}),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-09 17:11:25 +00:00
|
|
|
impl ThreadSafeState {
|
2019-04-11 14:58:31 +00:00
|
|
|
pub fn new(
|
|
|
|
flags: flags::DenoFlags,
|
|
|
|
argv_rest: Vec<String>,
|
|
|
|
dispatch_selector: ops::OpSelector,
|
2019-05-11 14:23:19 +00:00
|
|
|
progress: Progress,
|
2019-04-11 14:58:31 +00:00
|
|
|
) -> Self {
|
2019-04-17 13:25:51 +00:00
|
|
|
let custom_root = env::var("DENO_DIR").map(String::into).ok();
|
2019-03-14 23:17:52 +00:00
|
|
|
|
2019-04-08 21:10:00 +00:00
|
|
|
let (worker_in_tx, worker_in_rx) = async_mpsc::channel::<Buf>(1);
|
|
|
|
let (worker_out_tx, worker_out_rx) = async_mpsc::channel::<Buf>(1);
|
|
|
|
let internal_channels = (worker_out_tx, worker_in_rx);
|
|
|
|
let external_channels = (worker_in_tx, worker_out_rx);
|
|
|
|
let resource = resources::add_worker(external_channels);
|
|
|
|
|
2019-04-29 14:58:31 +00:00
|
|
|
// take the passed flag and resolve the file name relative to the cwd
|
|
|
|
let config_file = match &flags.config_path {
|
|
|
|
Some(config_file_name) => {
|
|
|
|
debug!("Compiler config file: {}", config_file_name);
|
|
|
|
let cwd = std::env::current_dir().unwrap();
|
|
|
|
Some(cwd.join(config_file_name))
|
|
|
|
}
|
|
|
|
_ => None,
|
|
|
|
};
|
|
|
|
|
|
|
|
// Convert the PathBuf to a canonicalized string. This is needed by the
|
|
|
|
// compiler to properly deal with the configuration.
|
|
|
|
let config_path = match &config_file {
|
|
|
|
Some(config_file) => Some(
|
|
|
|
config_file
|
|
|
|
.canonicalize()
|
|
|
|
.unwrap()
|
|
|
|
.to_str()
|
|
|
|
.unwrap()
|
|
|
|
.to_owned(),
|
|
|
|
),
|
|
|
|
_ => None,
|
|
|
|
};
|
|
|
|
|
|
|
|
// Load the contents of the configuration file
|
|
|
|
let config = match &config_file {
|
|
|
|
Some(config_file) => {
|
|
|
|
debug!("Attempt to load config: {}", config_file.to_str().unwrap());
|
|
|
|
match fs::read(&config_file) {
|
|
|
|
Ok(config_data) => Some(config_data.to_owned()),
|
|
|
|
_ => panic!(
|
|
|
|
"Error retrieving compiler config file at \"{}\"",
|
|
|
|
config_file.to_str().unwrap()
|
|
|
|
),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ => None,
|
|
|
|
};
|
|
|
|
|
2019-06-09 13:08:20 +00:00
|
|
|
let dir =
|
|
|
|
deno_dir::DenoDir::new(custom_root, &config, progress.clone()).unwrap();
|
|
|
|
|
|
|
|
let main_module: Option<String> = if argv_rest.len() <= 1 {
|
|
|
|
None
|
|
|
|
} else {
|
|
|
|
let specifier = argv_rest[1].clone();
|
|
|
|
let referrer = ".";
|
|
|
|
// TODO: does this really have to be resolved by DenoDir?
|
|
|
|
// Maybe we can call `resolve_module_spec`
|
|
|
|
match dir.resolve_module_url(&specifier, referrer) {
|
|
|
|
Ok(url) => Some(url.to_string()),
|
|
|
|
Err(e) => {
|
|
|
|
debug!("Potentially swallowed error {}", e);
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let mut import_map = None;
|
|
|
|
if let Some(file_name) = &flags.import_map_path {
|
|
|
|
let base_url = match &main_module {
|
|
|
|
Some(url) => url,
|
|
|
|
None => unreachable!(),
|
|
|
|
};
|
|
|
|
|
|
|
|
match ImportMap::load(base_url, file_name) {
|
|
|
|
Ok(map) => import_map = Some(map),
|
|
|
|
Err(err) => {
|
|
|
|
println!("{:?}", err);
|
|
|
|
panic!("Error parsing import map");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-09 17:11:25 +00:00
|
|
|
ThreadSafeState(Arc::new(State {
|
2019-06-09 13:08:20 +00:00
|
|
|
main_module,
|
|
|
|
dir,
|
2019-03-14 23:17:52 +00:00
|
|
|
argv: argv_rest,
|
2019-03-20 00:55:59 +00:00
|
|
|
permissions: DenoPermissions::from_flags(&flags),
|
2019-03-14 23:17:52 +00:00
|
|
|
flags,
|
2019-04-29 14:58:31 +00:00
|
|
|
config,
|
|
|
|
config_path,
|
2019-06-09 13:08:20 +00:00
|
|
|
import_map,
|
2019-03-14 23:17:52 +00:00
|
|
|
metrics: Metrics::default(),
|
2019-04-08 21:10:00 +00:00
|
|
|
worker_channels: Mutex::new(internal_channels),
|
2019-03-14 23:17:52 +00:00
|
|
|
global_timer: Mutex::new(GlobalTimer::new()),
|
2019-04-01 19:09:59 +00:00
|
|
|
workers: Mutex::new(UserWorkerTable::new()),
|
2019-04-08 20:22:40 +00:00
|
|
|
start_time: Instant::now(),
|
2019-04-08 21:10:00 +00:00
|
|
|
resource,
|
2019-04-11 14:58:31 +00:00
|
|
|
dispatch_selector,
|
2019-05-11 14:23:19 +00:00
|
|
|
progress,
|
2019-05-20 16:06:57 +00:00
|
|
|
compiled: Mutex::new(HashSet::new()),
|
2019-04-09 17:11:25 +00:00
|
|
|
}))
|
2019-03-14 23:17:52 +00:00
|
|
|
}
|
|
|
|
|
2019-04-01 19:09:59 +00:00
|
|
|
/// Read main module from argv
|
2019-03-14 23:17:52 +00:00
|
|
|
pub fn main_module(&self) -> Option<String> {
|
2019-06-09 13:08:20 +00:00
|
|
|
match &self.main_module {
|
|
|
|
Some(url) => Some(url.to_string()),
|
|
|
|
None => None,
|
2019-03-14 23:17:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-20 16:06:57 +00:00
|
|
|
pub fn mark_compiled(&self, module_id: &str) {
|
|
|
|
let mut c = self.compiled.lock().unwrap();
|
|
|
|
c.insert(module_id.to_string());
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn has_compiled(&self, module_id: &str) -> bool {
|
|
|
|
let c = self.compiled.lock().unwrap();
|
|
|
|
c.contains(module_id)
|
|
|
|
}
|
|
|
|
|
2019-03-20 00:55:59 +00:00
|
|
|
#[inline]
|
|
|
|
pub fn check_read(&self, filename: &str) -> DenoResult<()> {
|
|
|
|
self.permissions.check_read(filename)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
pub fn check_write(&self, filename: &str) -> DenoResult<()> {
|
|
|
|
self.permissions.check_write(filename)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
pub fn check_env(&self) -> DenoResult<()> {
|
|
|
|
self.permissions.check_env()
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
2019-05-08 23:20:30 +00:00
|
|
|
pub fn check_net(&self, host_and_port: &str) -> DenoResult<()> {
|
|
|
|
self.permissions.check_net(host_and_port)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
pub fn check_net_url(&self, url: url::Url) -> DenoResult<()> {
|
|
|
|
self.permissions.check_net_url(url)
|
2019-03-20 00:55:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
pub fn check_run(&self) -> DenoResult<()> {
|
|
|
|
self.permissions.check_run()
|
|
|
|
}
|
|
|
|
|
2019-03-14 23:17:52 +00:00
|
|
|
#[cfg(test)]
|
2019-06-08 18:42:28 +00:00
|
|
|
pub fn mock(argv: Vec<String>) -> ThreadSafeState {
|
2019-04-21 15:34:18 +00:00
|
|
|
ThreadSafeState::new(
|
|
|
|
flags::DenoFlags::default(),
|
|
|
|
argv,
|
|
|
|
ops::op_selector_std,
|
2019-05-11 14:23:19 +00:00
|
|
|
Progress::new(),
|
2019-04-21 15:34:18 +00:00
|
|
|
)
|
2019-03-14 23:17:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn metrics_op_dispatched(
|
|
|
|
&self,
|
|
|
|
bytes_sent_control: usize,
|
|
|
|
bytes_sent_data: usize,
|
|
|
|
) {
|
|
|
|
self.metrics.ops_dispatched.fetch_add(1, Ordering::SeqCst);
|
|
|
|
self
|
|
|
|
.metrics
|
|
|
|
.bytes_sent_control
|
|
|
|
.fetch_add(bytes_sent_control, Ordering::SeqCst);
|
|
|
|
self
|
|
|
|
.metrics
|
|
|
|
.bytes_sent_data
|
|
|
|
.fetch_add(bytes_sent_data, Ordering::SeqCst);
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn metrics_op_completed(&self, bytes_received: usize) {
|
|
|
|
self.metrics.ops_completed.fetch_add(1, Ordering::SeqCst);
|
|
|
|
self
|
|
|
|
.metrics
|
|
|
|
.bytes_received
|
|
|
|
.fetch_add(bytes_received, Ordering::SeqCst);
|
|
|
|
}
|
|
|
|
}
|
2019-04-09 17:11:25 +00:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn thread_safe() {
|
|
|
|
fn f<S: Send + Sync>(_: S) {}
|
2019-06-08 18:42:28 +00:00
|
|
|
f(ThreadSafeState::mock(vec![
|
|
|
|
String::from("./deno"),
|
|
|
|
String::from("hello.js"),
|
|
|
|
]));
|
2019-04-09 17:11:25 +00:00
|
|
|
}
|