development/deno
development/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno synced 2024-08-25 02:57:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rationalise compiler ops (#1740)

Browse source
This commit is contained in:
Kitson Kelly 2019-02-19 02:42:15 +11:00 committed by Ryan Dahl
parent 9e942f30b9
commit 54094c7510
12 changed files with 230 additions and 273 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
js/compiler.ts
View file

@ -55,8 +55,7 @@ interface CompilerLookup {
* easily mocked.
*/
interface Os {
codeCache: typeof os.codeCache;
codeFetch: typeof os.codeFetch;
fetchModuleMetaData: typeof os.fetchModuleMetaData;
exit: typeof os.exit;
}
@ -251,7 +250,10 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
} else {
// We query Rust with a CodeFetch message. It will load the sourceCode,
// and if there is any outputCode cached, will return that as well.
const fetchResponse = this._os.codeFetch(moduleSpecifier, containingFile);
const fetchResponse = this._os.fetchModuleMetaData(
moduleSpecifier,
containingFile
);
moduleId = fetchResponse.moduleName;
fileName = fetchResponse.filename;
mediaType = fetchResponse.mediaType;
@ -307,22 +309,26 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
// Deno specific compiler API
/** Retrieve the output of the TypeScript compiler for a given module and
* cache the result.
/** Retrieve the output of the TypeScript compiler for a given module.
*/
compile(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleMetaData {
): { outputCode: OutputCode; sourceMap: SourceMap } {
this._log("compiler.compile", { moduleSpecifier, containingFile });
const moduleMetaData = this._resolveModule(moduleSpecifier, containingFile);
const { fileName, mediaType, moduleId, sourceCode } = moduleMetaData;
this._scriptFileNames = [fileName];
console.warn("Compiling", moduleId);
let outputCode: string;
let sourceMap = "";
// Instead of using TypeScript to transpile JSON modules, we will just do
// it directly.
if (mediaType === msg.MediaType.Json) {
moduleMetaData.outputCode = jsonEsmTemplate(sourceCode, fileName);
outputCode = moduleMetaData.outputCode = jsonEsmTemplate(
sourceCode,
fileName
);
} else {
const service = this._service;
assert(
@ -373,20 +379,14 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
outputFile.name.endsWith(".js"),
"Expected second emitted file to be JavaScript"
);
moduleMetaData.outputCode = `${
outputCode = moduleMetaData.outputCode = `${
outputFile.text
}\n//# sourceURL=${fileName}`;
moduleMetaData.sourceMap = sourceMapFile.text;
sourceMap = moduleMetaData.sourceMap = sourceMapFile.text;
}
moduleMetaData.scriptVersion = "1";
this._os.codeCache(
fileName,
sourceCode,
moduleMetaData.outputCode,
moduleMetaData.sourceMap
);
return moduleMetaData;
return { outputCode, sourceMap };
}
// TypeScript Language Service and Format Diagnostic Host API
@ -532,9 +532,9 @@ window.compilerMain = function compilerMain() {
const json = decoder.decode(data);
const { specifier, referrer } = JSON.parse(json) as CompilerLookup;
const moduleMetaData = compiler.compile(specifier, referrer);
const result = compiler.compile(specifier, referrer);
const responseJson = JSON.stringify(moduleMetaData);
const responseJson = JSON.stringify(result);
const response = encoder.encode(responseJson);
postMessage(response);
};

68
js/os.ts
View file

@ -3,6 +3,7 @@ import * as msg from "gen/msg_generated";
import { handleAsyncMsgFromRust, sendSync } from "./dispatch";
import * as flatbuffers from "./flatbuffers";
import { libdeno } from "./libdeno";
import { TextDecoder } from "./text_encoding";
import { assert } from "./util";
import * as util from "./util";
@ -27,7 +28,7 @@ export function setGlobals(
execPath = execPath_;
}
interface CodeInfo {
interface ResponseModuleMetaData {
moduleName: string | undefined;
filename: string | undefined;
mediaType: msg.MediaType;
@ -60,63 +61,42 @@ export function exit(exitCode = 0): never {
return util.unreachable();
}
const decoder = new TextDecoder();
// @internal
export function codeFetch(specifier: string, referrer: string): CodeInfo {
util.log("os.codeFetch", { specifier, referrer });
// Send CodeFetch message
export function fetchModuleMetaData(
specifier: string,
referrer: string
): ResponseModuleMetaData {
util.log("os.fetchModuleMetaData", { specifier, referrer });
// Send FetchModuleMetaData message
const builder = flatbuffers.createBuilder();
const specifier_ = builder.createString(specifier);
const referrer_ = builder.createString(referrer);
msg.CodeFetch.startCodeFetch(builder);
msg.CodeFetch.addSpecifier(builder, specifier_);
msg.CodeFetch.addReferrer(builder, referrer_);
const inner = msg.CodeFetch.endCodeFetch(builder);
const baseRes = sendSync(builder, msg.Any.CodeFetch, inner);
msg.FetchModuleMetaData.startFetchModuleMetaData(builder);
msg.FetchModuleMetaData.addSpecifier(builder, specifier_);
msg.FetchModuleMetaData.addReferrer(builder, referrer_);
const inner = msg.FetchModuleMetaData.endFetchModuleMetaData(builder);
const baseRes = sendSync(builder, msg.Any.FetchModuleMetaData, inner);
assert(baseRes != null);
assert(
msg.Any.CodeFetchRes === baseRes!.innerType(),
msg.Any.FetchModuleMetaDataRes === baseRes!.innerType(),
`base.innerType() unexpectedly is ${baseRes!.innerType()}`
);
const codeFetchRes = new msg.CodeFetchRes();
assert(baseRes!.inner(codeFetchRes) != null);
const fetchModuleMetaDataRes = new msg.FetchModuleMetaDataRes();
assert(baseRes!.inner(fetchModuleMetaDataRes) != null);
const dataArray = fetchModuleMetaDataRes.dataArray();
const sourceCode = dataArray ? decoder.decode(dataArray) : undefined;
// flatbuffers returns `null` for an empty value, this does not fit well with
// idiomatic TypeScript under strict null checks, so converting to `undefined`
return {
moduleName: codeFetchRes.moduleName() || undefined,
filename: codeFetchRes.filename() || undefined,
mediaType: codeFetchRes.mediaType(),
sourceCode: codeFetchRes.sourceCode() || undefined
moduleName: fetchModuleMetaDataRes.moduleName() || undefined,
filename: fetchModuleMetaDataRes.filename() || undefined,
mediaType: fetchModuleMetaDataRes.mediaType(),
sourceCode
};
}
// @internal
export function codeCache(
filename: string,
sourceCode: string,
outputCode: string,
sourceMap: string
): void {
util.log("os.codeCache", {
filename,
sourceCodeLength: sourceCode.length,
outputCodeLength: outputCode.length,
sourceMapLength: sourceMap.length
});
const builder = flatbuffers.createBuilder();
const filename_ = builder.createString(filename);
const sourceCode_ = builder.createString(sourceCode);
const outputCode_ = builder.createString(outputCode);
const sourceMap_ = builder.createString(sourceMap);
msg.CodeCache.startCodeCache(builder);
msg.CodeCache.addFilename(builder, filename_);
msg.CodeCache.addSourceCode(builder, sourceCode_);
msg.CodeCache.addOutputCode(builder, outputCode_);
msg.CodeCache.addSourceMap(builder, sourceMap_);
const inner = msg.CodeCache.endCodeCache(builder);
const baseRes = sendSync(builder, msg.Any.CodeCache, inner);
assert(baseRes == null); // Expect null or error.
}
function createEnv(inner: msg.EnvironRes): { [index: string]: string } {
const env: { [index: string]: string } = {};

123
src/compiler.rs
View file

@ -9,6 +9,7 @@ use crate::workers;
use futures::Future;
use serde_json;
use std::str;
use std::sync::Arc;
use std::sync::Mutex;
@ -19,66 +20,28 @@ lazy_static! {
// This corresponds to JS ModuleMetaData.
// TODO Rename one or the other so they correspond.
#[derive(Debug)]
pub struct CodeFetchOutput {
pub struct ModuleMetaData {
pub module_name: String,
pub filename: String,
pub media_type: msg::MediaType,
pub source_code: String,
pub source_code: Vec<u8>,
pub maybe_output_code_filename: Option<String>,
pub maybe_output_code: Option<String>,
pub maybe_output_code: Option<Vec<u8>>,
pub maybe_source_map_filename: Option<String>,
pub maybe_source_map: Option<String>,
pub maybe_source_map: Option<Vec<u8>>,
}
impl CodeFetchOutput {
impl ModuleMetaData {
pub fn js_source(&self) -> String {
if self.media_type == msg::MediaType::Json {
return format!("export default {};", self.source_code);
return format!(
"export default {};",
str::from_utf8(&self.source_code).unwrap()
);
}
match self.maybe_output_code {
None => self.source_code.clone(),
Some(ref output_code) => output_code.clone(),
}
}
}
impl CodeFetchOutput {
// TODO Use serde_derive? Use flatbuffers?
fn from_json(json_str: &str) -> Option<Self> {
match serde_json::from_str::<serde_json::Value>(json_str) {
Ok(serde_json::Value::Object(map)) => {
let module_name = match map["moduleId"].as_str() {
None => return None,
Some(s) => s.to_string(),
};
let filename = match map["fileName"].as_str() {
None => return None,
Some(s) => s.to_string(),
};
let source_code = match map["sourceCode"].as_str() {
None => return None,
Some(s) => s.to_string(),
};
let maybe_output_code =
map["outputCode"].as_str().map(|s| s.to_string());
let maybe_source_map = map["sourceMap"].as_str().map(|s| s.to_string());
Some(CodeFetchOutput {
module_name,
filename,
media_type: msg::MediaType::JavaScript, // TODO
source_code,
maybe_output_code_filename: None,
maybe_output_code,
maybe_source_map_filename: None,
maybe_source_map,
})
}
_ => None,
None => str::from_utf8(&self.source_code).unwrap().to_string(),
Some(ref output_code) => str::from_utf8(output_code).unwrap().to_string(),
}
}
}
@ -106,7 +69,8 @@ pub fn compile_sync(
parent_state: &Arc<IsolateState>,
specifier: &str,
referrer: &str,
) -> Option<CodeFetchOutput> {
module_meta_data: &ModuleMetaData,
) -> ModuleMetaData {
let req_msg = req(specifier, referrer);
let compiler = lazy_start(parent_state);
@ -118,7 +82,25 @@ pub fn compile_sync(
let res_msg = recv_future.wait().unwrap().unwrap();
let res_json = std::str::from_utf8(&res_msg).unwrap();
CodeFetchOutput::from_json(res_json)
match serde_json::from_str::<serde_json::Value>(res_json) {
Ok(serde_json::Value::Object(map)) => ModuleMetaData {
module_name: module_meta_data.module_name.clone(),
filename: module_meta_data.filename.clone(),
media_type: module_meta_data.media_type,
source_code: module_meta_data.source_code.clone(),
maybe_output_code: match map["outputCode"].as_str() {
Some(str) => Some(str.as_bytes().to_owned()),
_ => None,
},
maybe_output_code_filename: None,
maybe_source_map: match map["sourceMap"].as_str() {
Some(str) => Some(str.as_bytes().to_owned()),
_ => None,
},
maybe_source_map_filename: None,
},
_ => panic!("error decoding compiler response"),
}
}
#[cfg(test)]
@ -133,28 +115,23 @@ mod tests {
let specifier = "./tests/002_hello.ts";
let referrer = cwd_string + "/";
let cfo =
compile_sync(&IsolateState::mock(), specifier, &referrer).unwrap();
let output_code = cfo.maybe_output_code.unwrap();
assert!(output_code.starts_with("console.log(\"Hello World\");"));
}
let mut out = ModuleMetaData {
module_name: "xxx".to_owned(),
filename: "/tests/002_hello.ts".to_owned(),
media_type: msg::MediaType::TypeScript,
source_code: "console.log(\"Hello World\");".as_bytes().to_owned(),
maybe_output_code_filename: None,
maybe_output_code: None,
maybe_source_map_filename: None,
maybe_source_map: None,
};
#[test]
fn code_fetch_output_from_json() {
let json = r#"{
"moduleId":"/Users/rld/src/deno/tests/002_hello.ts",
"fileName":"/Users/rld/src/deno/tests/002_hello.ts",
"mediaType":1,
"sourceCode":"console.log(\"Hello World\");\n",
"outputCode":"yyy",
"sourceMap":"xxx",
"scriptVersion":"1"
}"#;
let actual = CodeFetchOutput::from_json(json).unwrap();
assert_eq!(actual.filename, "/Users/rld/src/deno/tests/002_hello.ts");
assert_eq!(actual.module_name, "/Users/rld/src/deno/tests/002_hello.ts");
assert_eq!(actual.source_code, "console.log(\"Hello World\");\n");
assert_eq!(actual.maybe_output_code, Some("yyy".to_string()));
assert_eq!(actual.maybe_source_map, Some("xxx".to_string()));
out = compile_sync(&IsolateState::mock(), specifier, &referrer, &mut out);
assert!(
out
.maybe_output_code
.unwrap()
.starts_with("console.log(\"Hello World\");".as_bytes())
);
}
}

176
src/deno_dir.rs
View file

@ -1,5 +1,5 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use crate::compiler::CodeFetchOutput;
use crate::compiler::ModuleMetaData;
use crate::errors;
use crate::errors::DenoError;
use crate::errors::DenoResult;
@ -18,6 +18,7 @@ use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::result::Result;
use std::str;
use url;
use url::Url;
@ -106,7 +107,7 @@ impl DenoDir {
pub fn cache_path(
self: &Self,
filename: &str,
source_code: &str,
source_code: &[u8],
) -> (PathBuf, PathBuf) {
let cache_key = source_code_hash(filename, source_code, version::DENO);
(
@ -118,27 +119,25 @@ impl DenoDir {
fn load_cache(
self: &Self,
filename: &str,
source_code: &str,
) -> Result<(String, String), std::io::Error> {
source_code: &[u8],
) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
let (output_code, source_map) = self.cache_path(filename, source_code);
debug!(
"load_cache code: {} map: {}",
output_code.display(),
source_map.display()
);
let read_output_code = fs::read_to_string(&output_code)?;
let read_source_map = fs::read_to_string(&source_map)?;
let read_output_code = fs::read(&output_code)?;
let read_source_map = fs::read(&source_map)?;
Ok((read_output_code, read_source_map))
}
pub fn code_cache(
self: &Self,
filename: &str,
source_code: &str,
output_code: &str,
source_map: &str,
module_meta_data: &ModuleMetaData,
) -> std::io::Result<()> {
let (cache_path, source_map_path) = self.cache_path(filename, source_code);
let (cache_path, source_map_path) = self
.cache_path(&module_meta_data.filename, &module_meta_data.source_code);
// TODO(ry) This is a race condition w.r.t to exists() -- probably should
// create the file in exclusive mode. A worry is what might happen is there
// are two processes and one reads the cache file while the other is in the
@ -146,8 +145,14 @@ impl DenoDir {
if cache_path.exists() && source_map_path.exists() {
Ok(())
} else {
fs::write(cache_path, output_code.as_bytes())?;
fs::write(source_map_path, source_map.as_bytes())?;
match &module_meta_data.maybe_output_code {
Some(output_code) => fs::write(cache_path, output_code),
_ => Ok(()),
}?;
match &module_meta_data.maybe_source_map {
Some(source_map) => fs::write(source_map_path, source_map),
_ => Ok(()),
}?;
Ok(())
}
}
@ -158,7 +163,7 @@ impl DenoDir {
self: &Self,
module_name: &str,
filename: &str,
) -> DenoResult<Option<CodeFetchOutput>> {
) -> DenoResult<Option<ModuleMetaData>> {
let p = Path::new(&filename);
// We write a special ".mime" file into the `.deno/deps` directory along side the
// cached file, containing just the media type.
@ -188,11 +193,11 @@ impl DenoDir {
{
deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?
}
return Ok(Some(CodeFetchOutput {
return Ok(Some(ModuleMetaData {
module_name: module_name.to_string(),
filename: filename.to_string(),
media_type: map_content_type(&p, Some(&content_type)),
source_code: source,
source_code: source.as_bytes().to_owned(),
maybe_output_code_filename: None,
maybe_output_code: None,
maybe_source_map_filename: None,
@ -209,7 +214,7 @@ impl DenoDir {
self: &Self,
module_name: &str,
filename: &str,
) -> DenoResult<Option<CodeFetchOutput>> {
) -> DenoResult<Option<ModuleMetaData>> {
let p = Path::new(&filename);
let media_type_filename = [&filename, ".mime"].concat();
let mt = Path::new(&media_type_filename);
@ -221,7 +226,7 @@ impl DenoDir {
return Err(e.into());
}
}
Ok(c) => String::from_utf8(c).unwrap(),
Ok(c) => c,
};
// .mime file might not exists
// this is okay for local source: maybe_content_type_str will be None
@ -229,7 +234,7 @@ impl DenoDir {
// Option<String> -> Option<&str>
let maybe_content_type_str =
maybe_content_type_string.as_ref().map(String::as_str);
Ok(Some(CodeFetchOutput {
Ok(Some(ModuleMetaData {
module_name: module_name.to_string(),
filename: filename.to_string(),
media_type: map_content_type(&p, maybe_content_type_str),
@ -246,7 +251,7 @@ impl DenoDir {
self: &Self,
module_name: &str,
filename: &str,
) -> DenoResult<CodeFetchOutput> {
) -> DenoResult<ModuleMetaData> {
let is_module_remote = is_remote(module_name);
// We try fetch local. Two cases:
// 1. This is a remote module, but no reload provided
@ -290,12 +295,15 @@ impl DenoDir {
)))
}
pub fn code_fetch(
pub fn fetch_module_meta_data(
self: &Self,
specifier: &str,
referrer: &str,
) -> Result<CodeFetchOutput, errors::DenoError> {
debug!("code_fetch. specifier {} referrer {}", specifier, referrer);
) -> Result<ModuleMetaData, errors::DenoError> {
debug!(
"fetch_module_meta_data. specifier {} referrer {}",
specifier, referrer
);
let (module_name, filename) = self.resolve_module(specifier, referrer)?;
@ -318,7 +326,9 @@ impl DenoDir {
}
};
out.source_code = filter_shebang(out.source_code);
if out.source_code.starts_with("#!".as_bytes()) {
out.source_code = filter_shebang(out.source_code);
}
if out.media_type != msg::MediaType::TypeScript {
return Ok(out);
@ -330,8 +340,7 @@ impl DenoDir {
let mut maybe_source_map = None;
if !self.recompile {
let result =
self.load_cache(out.filename.as_str(), out.source_code.as_str());
let result = self.load_cache(out.filename.as_str(), &out.source_code);
match result {
Err(err) => {
if err.kind() == std::io::ErrorKind::NotFound {
@ -347,7 +356,7 @@ impl DenoDir {
}
}
Ok(CodeFetchOutput {
Ok(ModuleMetaData {
module_name: out.module_name,
filename: out.filename,
media_type: out.media_type,
@ -457,8 +466,8 @@ impl DenoDir {
}
impl SourceMapGetter for DenoDir {
fn get_source_map(&self, script_name: &str) -> Option<String> {
match self.code_fetch(script_name, ".") {
fn get_source_map(&self, script_name: &str) -> Option<Vec<u8>> {
match self.fetch_module_meta_data(script_name, ".") {
Err(_e) => None,
Ok(out) => match out.maybe_source_map {
None => None,
@ -487,13 +496,13 @@ fn get_cache_filename(basedir: &Path, url: &Url) -> PathBuf {
fn source_code_hash(
filename: &str,
source_code: &str,
source_code: &[u8],
version: &str,
) -> String {
let mut ctx = ring::digest::Context::new(&ring::digest::SHA1);
ctx.update(version.as_bytes());
ctx.update(filename.as_bytes());
ctx.update(source_code.as_bytes());
ctx.update(source_code);
let digest = ctx.finish();
let mut out = String::new();
// TODO There must be a better way to do this...
@ -559,15 +568,13 @@ fn map_content_type(path: &Path, content_type: Option<&str>) -> msg::MediaType {
}
}
fn filter_shebang(code: String) -> String {
if !code.starts_with("#!") {
return code;
}
if let Some(i) = code.find('\n') {
let (_, rest) = code.split_at(i);
String::from(rest)
fn filter_shebang(bytes: Vec<u8>) -> Vec<u8> {
let string = str::from_utf8(&bytes).unwrap();
if let Some(i) = string.find('\n') {
let (_, rest) = string.split_at(i);
rest.as_bytes().to_owned()
} else {
String::from("")
Vec::new()
}
}
@ -613,7 +620,7 @@ mod tests {
fn test_cache_path() {
let (temp_dir, deno_dir) = test_setup(false, false);
let filename = "hello.js";
let source_code = "1+2";
let source_code = "1+2".as_bytes();
let hash = source_code_hash(filename, source_code, version::DENO);
assert_eq!(
(
@ -629,41 +636,52 @@ mod tests {
let (_temp_dir, deno_dir) = test_setup(false, false);
let filename = "hello.js";
let source_code = "1+2";
let output_code = "1+2 // output code";
let source_map = "{}";
let source_code = "1+2".as_bytes();
let output_code = "1+2 // output code".as_bytes();
let source_map = "{}".as_bytes();
let hash = source_code_hash(filename, source_code, version::DENO);
let (cache_path, source_map_path) =
deno_dir.cache_path(filename, source_code);
assert!(cache_path.ends_with(format!("gen/{}.js", hash)));
assert!(source_map_path.ends_with(format!("gen/{}.js.map", hash)));
let r = deno_dir.code_cache(filename, source_code, output_code, source_map);
let out = ModuleMetaData {
filename: filename.to_owned(),
source_code: source_code.to_owned(),
module_name: "hello.js".to_owned(),
media_type: msg::MediaType::TypeScript,
maybe_output_code: Some(output_code.to_owned()),
maybe_output_code_filename: None,
maybe_source_map: Some(source_map.to_owned()),
maybe_source_map_filename: None,
};
let r = deno_dir.code_cache(&out);
r.expect("code_cache error");
assert!(cache_path.exists());
assert_eq!(output_code, fs::read_to_string(&cache_path).unwrap());
assert_eq!(output_code.to_owned(), fs::read(&cache_path).unwrap());
}
#[test]
fn test_source_code_hash() {
assert_eq!(
"7e44de2ed9e0065da09d835b76b8d70be503d276",
source_code_hash("hello.ts", "1+2", "0.2.11")
source_code_hash("hello.ts", "1+2".as_bytes(), "0.2.11")
);
// Different source_code should result in different hash.
assert_eq!(
"57033366cf9db1ef93deca258cdbcd9ef5f4bde1",
source_code_hash("hello.ts", "1", "0.2.11")
source_code_hash("hello.ts", "1".as_bytes(), "0.2.11")
);
// Different filename should result in different hash.
assert_eq!(
"19657f90b5b0540f87679e2fb362e7bd62b644b0",
source_code_hash("hi.ts", "1+2", "0.2.11")
source_code_hash("hi.ts", "1+2".as_bytes(), "0.2.11")
);
// Different version should result in different hash.
assert_eq!(
"e2b4b7162975a02bf2770f16836eb21d5bcb8be1",
source_code_hash("hi.ts", "1+2", "0.2.0")
source_code_hash("hi.ts", "1+2".as_bytes(), "0.2.0")
);
}
@ -686,8 +704,8 @@ mod tests {
assert!(result.is_ok());
let r = result.unwrap();
assert_eq!(
&(r.source_code),
"export { printHello } from \"./print_hello.ts\";\n"
r.source_code,
"export { printHello } from \"./print_hello.ts\";\n".as_bytes()
);
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// Should not create .mime file due to matching ext
@ -699,8 +717,8 @@ mod tests {
assert!(result2.is_ok());
let r2 = result2.unwrap();
assert_eq!(
&(r2.source_code),
"export { printHello } from \"./print_hello.ts\";\n"
r2.source_code,
"export { printHello } from \"./print_hello.ts\";\n".as_bytes()
);
// If get_source_code does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .mime due to no http fetch)
@ -717,7 +735,8 @@ mod tests {
let result3 = deno_dir.get_source_code(module_name, &filename);
assert!(result3.is_ok());
let r3 = result3.unwrap();
let expected3 = "export { printHello } from \"./print_hello.ts\";\n";
let expected3 =
"export { printHello } from \"./print_hello.ts\";\n".as_bytes();
assert_eq!(r3.source_code, expected3);
// Now the old .mime file should have gone! Resolved back to TypeScript
assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
@ -743,7 +762,7 @@ mod tests {
println!("module_name {} filename {}", module_name, filename);
assert!(result.is_ok());
let r = result.unwrap();
let expected = "export const loaded = true;\n";
let expected = "export const loaded = true;\n".as_bytes();
assert_eq!(r.source_code, expected);
// Mismatch ext with content type, create .mime
assert_eq!(&(r.media_type), &msg::MediaType::JavaScript);
@ -757,7 +776,7 @@ mod tests {
let result2 = deno_dir.get_source_code(module_name, &filename);
assert!(result2.is_ok());
let r2 = result2.unwrap();
let expected2 = "export const loaded = true;\n";
let expected2 = "export const loaded = true;\n".as_bytes();
assert_eq!(r2.source_code, expected2);
// If get_source_code does not call remote, this should be TypeScript
// as we modified before! (we do not overwrite .mime due to no http fetch)
@ -774,7 +793,7 @@ mod tests {
let result3 = deno_dir.get_source_code(module_name, &filename);
assert!(result3.is_ok());
let r3 = result3.unwrap();
let expected3 = "export const loaded = true;\n";
let expected3 = "export const loaded = true;\n".as_bytes();
assert_eq!(r3.source_code, expected3);
// Now the old .mime file should be overwritten back to JavaScript!
// (due to http fetch)
@ -805,7 +824,7 @@ mod tests {
let result = deno_dir.fetch_remote_source(module_name, &filename);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(&(r.source_code), "export const loaded = true;\n");
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// matching ext, no .mime file created
assert!(fs::read_to_string(&mime_file_name).is_err());
@ -815,7 +834,7 @@ mod tests {
let result2 = deno_dir.fetch_local_source(module_name, &filename);
assert!(result2.is_ok());
let r2 = result2.unwrap().unwrap();
assert_eq!(&(r2.source_code), "export const loaded = true;\n");
assert_eq!(r2.source_code, "export const loaded = true;\n".as_bytes());
// Not MediaType::TypeScript due to .mime modification
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
});
@ -838,7 +857,7 @@ mod tests {
let result = deno_dir.fetch_remote_source(module_name, &filename);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(&(r.source_code), "export const loaded = true;\n");
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// no ext, should create .mime file
assert_eq!(
@ -857,7 +876,7 @@ mod tests {
let result_2 = deno_dir.fetch_remote_source(module_name_2, &filename_2);
assert!(result_2.is_ok());
let r2 = result_2.unwrap().unwrap();
assert_eq!(&(r2.source_code), "export const loaded = true;\n");
assert_eq!(r2.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
// mismatch ext, should create .mime file
assert_eq!(
@ -877,7 +896,7 @@ mod tests {
let result_3 = deno_dir.fetch_remote_source(module_name_3, &filename_3);
assert!(result_3.is_ok());
let r3 = result_3.unwrap().unwrap();
assert_eq!(&(r3.source_code), "export const loaded = true;\n");
assert_eq!(r3.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
// unknown ext, should create .mime file
assert_eq!(
@ -900,12 +919,12 @@ mod tests {
let result = deno_dir.fetch_local_source(module_name, &filename);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(&(r.source_code), "export const loaded = true;\n");
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
}
#[test]
fn test_code_fetch() {
fn test_fetch_module_meta_data() {
let (_temp_dir, deno_dir) = test_setup(false, false);
let cwd = std::env::current_dir().unwrap();
@ -914,20 +933,20 @@ mod tests {
// Test failure case.
let specifier = "hello.ts";
let referrer = add_root!("/baddir/badfile.ts");
let r = deno_dir.code_fetch(specifier, referrer);
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
assert!(r.is_err());
// Assuming cwd is the deno repo root.
let specifier = "./js/main.ts";
let referrer = cwd_string.as_str();
let r = deno_dir.code_fetch(specifier, referrer);
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
assert!(r.is_ok());
//let code_fetch_output = r.unwrap();
//println!("code_fetch_output {:?}", code_fetch_output);
//let fetch_module_meta_data_output = r.unwrap();
//println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
}
#[test]
fn test_code_fetch_1() {
fn test_fetch_module_meta_data_1() {
/*recompile ts file*/
let (_temp_dir, deno_dir) = test_setup(false, true);
@ -937,13 +956,13 @@ mod tests {
// Test failure case.
let specifier = "hello.ts";
let referrer = add_root!("/baddir/badfile.ts");
let r = deno_dir.code_fetch(specifier, referrer);
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
assert!(r.is_err());
// Assuming cwd is the deno repo root.
let specifier = "./js/main.ts";
let referrer = cwd_string.as_str();
let r = deno_dir.code_fetch(specifier, referrer);
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
assert!(r.is_ok());
}
@ -1321,11 +1340,14 @@ mod tests {
#[test]
fn test_filter_shebang() {
assert_eq!(filter_shebang("".to_string()), "");
assert_eq!(filter_shebang("#".to_string()), "#");
assert_eq!(filter_shebang("#!".to_string()), "");
assert_eq!(filter_shebang("#!\n\n".to_string()), "\n\n");
let code = "#!/usr/bin/env deno\nconsole.log('hello');\n".to_string();
assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n");
assert_eq!(filter_shebang("#!".as_bytes().to_owned()), "".as_bytes());
assert_eq!(
filter_shebang("#!\n\n".as_bytes().to_owned()),
"\n\n".as_bytes()
);
let code = "#!/usr/bin/env deno\nconsole.log('hello');\n"
.as_bytes()
.to_owned();
assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n".as_bytes());
}
}

23
src/isolate.rs
View file

@ -6,7 +6,7 @@
#![allow(dead_code)]
use crate::compiler::compile_sync;
use crate::compiler::CodeFetchOutput;
use crate::compiler::ModuleMetaData;
use crate::deno_dir;
use crate::errors::DenoError;
use crate::errors::DenoResult;
@ -323,8 +323,11 @@ impl Isolate {
debug!("mod_load_deps {} {}", i, name);
if !self.modules.borrow_mut().is_registered(&name) {
let out =
code_fetch_and_maybe_compile(&self.state, specifier, &referrer_name)?;
let out = fetch_module_meta_data_and_maybe_compile(
&self.state,
specifier,
&referrer_name,
)?;
let child_id =
self.mod_new(out.module_name.clone(), out.js_source())?;
@ -367,8 +370,9 @@ impl Isolate {
js_filename: &str,
is_prefetch: bool,
) -> Result<(), RustOrJsError> {
let out = code_fetch_and_maybe_compile(&self.state, js_filename, ".")
.map_err(RustOrJsError::from)?;
let out =
fetch_module_meta_data_and_maybe_compile(&self.state, js_filename, ".")
.map_err(RustOrJsError::from)?;
let id = self
.mod_new(out.module_name.clone(), out.js_source())
@ -470,19 +474,20 @@ impl Drop for Isolate {
}
}
fn code_fetch_and_maybe_compile(
fn fetch_module_meta_data_and_maybe_compile(
state: &Arc<IsolateState>,
specifier: &str,
referrer: &str,
) -> Result<CodeFetchOutput, DenoError> {
let mut out = state.dir.code_fetch(specifier, referrer)?;
) -> Result<ModuleMetaData, DenoError> {
let mut out = state.dir.fetch_module_meta_data(specifier, referrer)?;
if (out.media_type == msg::MediaType::TypeScript
&& out.maybe_output_code.is_none())
|| state.flags.recompile
{
debug!(">>>>> compile_sync START");
out = compile_sync(state, specifier, &referrer).unwrap();
out = compile_sync(state, specifier, &referrer, &out);
debug!(">>>>> compile_sync END");
state.dir.code_cache(&out)?;
}
Ok(out)
}

11
src/js_errors.rs
View file

@ -16,10 +16,11 @@ use source_map_mappings::Bias;
use source_map_mappings::Mappings;
use std::collections::HashMap;
use std::fmt;
use std::str;
pub trait SourceMapGetter {
/// Returns the raw source map file.
fn get_source_map(&self, script_name: &str) -> Option<String>;
fn get_source_map(&self, script_name: &str) -> Option<Vec<u8>>;
}
struct SourceMap {
@ -389,7 +390,9 @@ fn parse_map_string(
}
_ => match getter.get_source_map(script_name) {
None => None,
Some(raw_source_map) => SourceMap::from_json(&raw_source_map),
Some(raw_source_map) => {
SourceMap::from_json(str::from_utf8(&raw_source_map).unwrap())
}
},
}
}
@ -455,13 +458,13 @@ mod tests {
struct MockSourceMapGetter {}
impl SourceMapGetter for MockSourceMapGetter {
fn get_source_map(&self, script_name: &str) -> Option<String> {
fn get_source_map(&self, script_name: &str) -> Option<Vec<u8>> {
let s = match script_name {
"foo_bar.ts" => r#"{"sources": ["foo_bar.ts"], "mappings":";;;IAIA,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC"}"#,
"bar_baz.ts" => r#"{"sources": ["bar_baz.ts"], "mappings":";;;IAEA,CAAC,KAAK,IAAI,EAAE;QACV,MAAM,GAAG,GAAG,sDAAa,OAAO,2BAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IAEQ,QAAA,GAAG,GAAG,KAAK,CAAC;IAEzB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC"}"#,
_ => return None,
};
Some(s.to_string())
Some(s.as_bytes().to_owned())
}
}

2
src/modules.rs
View file

@ -170,7 +170,7 @@ pub fn print_file_info(
deno_dir: &DenoDir,
filename: String,
) {
let maybe_out = deno_dir.code_fetch(&filename, ".");
let maybe_out = deno_dir.fetch_module_meta_data(&filename, ".");
if maybe_out.is_err() {
println!("{}", maybe_out.unwrap_err());
return;

20
src/msg.fbs
View file

@ -6,9 +6,8 @@ union Any {
WorkerGetMessage,
WorkerGetMessageRes,
WorkerPostMessage,
CodeFetch,
CodeFetchRes,
CodeCache,
FetchModuleMetaData,
FetchModuleMetaDataRes,
SetTimeout,
Exit,
Environ,
@ -186,28 +185,19 @@ table WorkerPostMessage {
// data passed thru the zero-copy data parameter.
}
table CodeFetch {
table FetchModuleMetaData {
specifier: string;
referrer: string;
}
table CodeFetchRes {
table FetchModuleMetaDataRes {
// If it's a non-http module, moduleName and filename will be the same.
// For http modules, moduleName is its resolved http URL, and filename
// is the location of the locally downloaded source code.
module_name: string;
filename: string;
media_type: MediaType;
// TODO These should be [ubyte].
// See: https://github.com/denoland/deno/issues/1113
source_code: string;
}
table CodeCache {
filename: string;
source_code: string;
output_code: string;
source_map: string;
data: [ubyte];
}
table Chdir {

38
src/ops.rs
View file

@ -90,8 +90,7 @@ pub fn dispatch(
msg::Any::Chdir => op_chdir,
msg::Any::Chmod => op_chmod,
msg::Any::Close => op_close,
msg::Any::CodeCache => op_code_cache,
msg::Any::CodeFetch => op_code_fetch,
msg::Any::FetchModuleMetaData => op_fetch_module_meta_data,
msg::Any::CopyFile => op_copy_file,
msg::Any::Cwd => op_cwd,
msg::Any::Dial => op_dial,
@ -349,13 +348,13 @@ pub fn odd_future(err: DenoError) -> Box<Op> {
}
// https://github.com/denoland/deno/blob/golang/os.go#L100-L154
fn op_code_fetch(
fn op_fetch_module_meta_data(
state: &Arc<IsolateState>,
base: &msg::Base<'_>,
data: libdeno::deno_buf,
) -> Box<Op> {
assert_eq!(data.len(), 0);
let inner = base.inner_as_code_fetch().unwrap();
let inner = base.inner_as_fetch_module_meta_data().unwrap();
let cmd_id = base.cmd_id();
let specifier = inner.specifier().unwrap();
let referrer = inner.referrer().unwrap();
@ -364,46 +363,27 @@ fn op_code_fetch(
Box::new(futures::future::result(|| -> OpResult {
let builder = &mut FlatBufferBuilder::new();
let out = state.dir.code_fetch(specifier, referrer)?;
let msg_args = msg::CodeFetchResArgs {
let out = state.dir.fetch_module_meta_data(specifier, referrer)?;
let data_off = builder.create_vector(out.source_code.as_slice());
let msg_args = msg::FetchModuleMetaDataResArgs {
module_name: Some(builder.create_string(&out.module_name)),
filename: Some(builder.create_string(&out.filename)),
media_type: out.media_type,
source_code: Some(builder.create_string(&out.source_code)),
data: Some(data_off),
};
let inner = msg::CodeFetchRes::create(builder, &msg_args);
let inner = msg::FetchModuleMetaDataRes::create(builder, &msg_args);
Ok(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(inner.as_union_value()),
inner_type: msg::Any::CodeFetchRes,
inner_type: msg::Any::FetchModuleMetaDataRes,
..Default::default()
},
))
}()))
}
// https://github.com/denoland/deno/blob/golang/os.go#L156-L169
fn op_code_cache(
state: &Arc<IsolateState>,
base: &msg::Base<'_>,
data: libdeno::deno_buf,
) -> Box<Op> {
assert_eq!(data.len(), 0);
let inner = base.inner_as_code_cache().unwrap();
let filename = inner.filename().unwrap();
let source_code = inner.source_code().unwrap();
let output_code = inner.output_code().unwrap();
let source_map = inner.source_map().unwrap();
Box::new(futures::future::result(|| -> OpResult {
state
.dir
.code_cache(filename, source_code, output_code, source_map)?;
Ok(empty_buf())
}()))
}
fn op_chdir(
_state: &Arc<IsolateState>,
base: &msg::Base<'_>,

2
tests/error_004_missing_module.ts.out
View file

@ -4,7 +4,7 @@ Uncaught NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/
at maybeError ([WILDCARD]/js/errors.ts:[WILDCARD])
at maybeThrowError ([WILDCARD]/js/errors.ts:[WILDCARD])
at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD])
at codeFetch ([WILDCARD]/js/os.ts:[WILDCARD])
at fetchModuleMetaData ([WILDCARD]/js/os.ts:[WILDCARD])
at _resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD])
at resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD])
at compilerHost.resolveModuleNames ([WILDCARD]typescript.js:[WILDCARD])

2
tests/error_005_missing_dynamic_import.ts.out
View file

@ -3,7 +3,7 @@
at maybeError ([WILDCARD]/js/errors.ts:[WILDCARD])
at maybeThrowError ([WILDCARD]/js/errors.ts:[WILDCARD])
at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD])
at codeFetch ([WILDCARD]/js/os.ts:[WILDCARD])
at fetchModuleMetaData ([WILDCARD]/js/os.ts:[WILDCARD])
at _resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD])
at resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD])
at compilerHost.resolveModuleNames ([WILDCARD])

2
tests/error_006_import_ext_failure.ts.out
View file

@ -4,7 +4,7 @@ Uncaught NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests
at maybeError ([WILDCARD]/js/errors.ts:[WILDCARD])
at maybeThrowError ([WILDCARD]/js/errors.ts:[WILDCARD])
at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD])
at codeFetch ([WILDCARD]/js/os.ts:[WILDCARD])
at fetchModuleMetaData ([WILDCARD]/js/os.ts:[WILDCARD])
at _resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD])
at resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD])
at compilerHost.resolveModuleNames ([WILDCARD])