Do not encode files loaded from network as utf8 (#3856)

cli/file_fetcher.rs

@@ -446,8 +446,7 @@ impl SourceFileFetcher {
     let http_client = self.http_client.clone();
     // Single pass fetch, either yields code or yields redirect.
     let f = async move {
-      match http_util::fetch_string_once(http_client, &module_url, module_etag)
-        .await?
+      match http_util::fetch_once(http_client, &module_url, module_etag).await?
       {
         FetchOnceResult::NotModified => {
           let source_file =
@@ -526,7 +525,7 @@ impl SourceFileFetcher {
           let types_url = match media_type {
             msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
               &module_url,
-              source.as_bytes(),
+              &source,
               x_typescript_types.as_ref().map(String::as_str),
             ),
             _ => None,
@@ -536,7 +535,7 @@ impl SourceFileFetcher {
             url: module_url.clone(),
             filename: filepath,
             media_type,
-            source_code: source.as_bytes().to_owned(),
+            source_code: source,
             types_url,
           };
 
@@ -571,13 +570,13 @@ impl SourceFileFetcher {
   }
 
   /// Save contents of downloaded remote file in on-disk cache for subsequent access.
-  fn save_source_code(&self, url: &Url, source: &str) -> std::io::Result<()> {
+  fn save_source_code(&self, url: &Url, source: &[u8]) -> std::io::Result<()> {
     let cache_key = self.deps_cache.get_cache_filename(url);
 
     // May not exist. DON'T unwrap.
     let _ = self.deps_cache.remove(&cache_key);
 
-    self.deps_cache.set(&cache_key, source.as_bytes())
+    self.deps_cache.set(&cache_key, source)
   }
 
   /// Save headers related to source file to {filename}.headers.json file,
@@ -1934,7 +1933,7 @@ mod tests {
     // it again with the cache parameters turned off.
     // If the fetched content changes, the cached content is used.
     fetcher
-      .save_source_code(&module_url, "changed content")
+      .save_source_code(&module_url, b"changed content")
       .unwrap();
     let cached_source = fetcher
       .fetch_remote_source_async(&module_url, false, false, 1)

cli/http_util.rs

@@ -75,7 +75,7 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
 
 #[derive(Debug, PartialEq)]
 pub struct ResultPayload {
-  pub body: String,
+  pub body: Vec<u8>,
   pub content_type: Option<String>,
   pub etag: Option<String>,
   pub x_typescript_types: Option<String>,
@@ -93,7 +93,7 @@ pub enum FetchOnceResult {
 /// yields Code(ResultPayload).
 /// If redirect occurs, does not follow and
 /// yields Redirect(url).
-pub fn fetch_string_once(
+pub fn fetch_once(
   client: Client,
   url: &Url,
   cached_etag: Option<String>,
@@ -169,14 +169,14 @@ pub fn fetch_string_once(
         _ if content_encoding == "br" => {
           let full_bytes = response.bytes().await?;
           let mut decoder = BrotliDecoder::new(full_bytes.as_ref());
-          let mut body = String::new();
-          decoder.read_to_string(&mut body)?;
+          let mut body = vec![];
+          decoder.read_to_end(&mut body)?;
           body
         }
-        _ => response.text().await?,
+        _ => response.bytes().await?.to_vec(),
       }
     } else {
-      body = response.text().await?;
+      body = response.bytes().await?.to_vec();
     }
 
     return Ok(FetchOnceResult::Code(ResultPayload {
@@ -277,7 +277,7 @@ mod tests {
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
     let client = create_http_client();
-    let result = fetch_string_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None).await;
     if let Ok(FetchOnceResult::Code(payload)) = result {
       assert!(!payload.body.is_empty());
       assert_eq!(payload.content_type, Some("application/json".to_string()));
@@ -298,9 +298,12 @@ mod tests {
     )
     .unwrap();
     let client = create_http_client();
-    let result = fetch_string_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None).await;
     if let Ok(FetchOnceResult::Code(payload)) = result {
-      assert_eq!(payload.body, "console.log('gzip')");
+      assert_eq!(
+        String::from_utf8(payload.body).unwrap(),
+        "console.log('gzip')"
+      );
       assert_eq!(
         payload.content_type,
         Some("application/javascript".to_string())
@@ -318,7 +321,7 @@ mod tests {
     let http_server_guard = crate::test_util::http_server();
     let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
     let client = create_http_client();
-    let result = fetch_string_once(client.clone(), &url, None).await;
+    let result = fetch_once(client.clone(), &url, None).await;
     if let Ok(FetchOnceResult::Code(ResultPayload {
       body,
       content_type,
@@ -327,7 +330,7 @@ mod tests {
     })) = result
     {
       assert!(!body.is_empty());
-      assert_eq!(body, "console.log('etag')");
+      assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
       assert_eq!(content_type, Some("application/typescript".to_string()));
       assert_eq!(etag, Some("33a64df551425fcc55e".to_string()));
       assert_eq!(x_typescript_types, None);
@@ -336,8 +339,7 @@ mod tests {
     }
 
     let res =
-      fetch_string_once(client, &url, Some("33a64df551425fcc55e".to_string()))
-        .await;
+      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string())).await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
 
     drop(http_server_guard);
@@ -352,10 +354,13 @@ mod tests {
     )
     .unwrap();
     let client = create_http_client();
-    let result = fetch_string_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None).await;
     if let Ok(FetchOnceResult::Code(payload)) = result {
       assert!(!payload.body.is_empty());
-      assert_eq!(payload.body, "console.log('brotli');");
+      assert_eq!(
+        String::from_utf8(payload.body).unwrap(),
+        "console.log('brotli');"
+      );
       assert_eq!(
         payload.content_type,
         Some("application/javascript".to_string())
@@ -369,7 +374,7 @@ mod tests {
   }
 
   #[tokio::test]
-  async fn test_fetch_string_once_with_redirect() {
+  async fn test_fetch_once_with_redirect() {
     let http_server_guard = crate::test_util::http_server();
     // Relies on external http server. See tools/http_server.py
     let url =
@@ -378,7 +383,7 @@ mod tests {
     let target_url =
       Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
     let client = create_http_client();
-    let result = fetch_string_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None).await;
     if let Ok(FetchOnceResult::Redirect(url)) = result {
       assert_eq!(url, target_url);
     } else {

cli/tests/055_import_wasm_via_network.ts

@@ -0,0 +1,2 @@
+import * as wasm from "./055_import_wasm_via_network.wasm";
+console.log(wasm);

cli/tests/055_import_wasm_via_network.ts.out

@@ -0,0 +1 @@
+{ __data_end, __dso_handle, __global_base, __heap_base, __wasm_call_ctors, add, memory }

cli/tests/integration_tests.rs

@@ -834,6 +834,12 @@ itest!(_053_import_compression {
   http_server: true,
 });
 
+itest!(import_wasm_via_network {
+  args: "run --reload http://127.0.0.1:4545/cli/tests/055_import_wasm_via_network.ts",
+  output: "055_import_wasm_via_network.ts.out",
+  http_server: true,
+});
+
 mod util {
   use deno::colors::strip_ansi_codes;
   pub use deno::test_util::*;