feat(web): Implement TextDecoderStream and TextEncoderStream (#10842)

extensions/web/08_text_encoding.js

@@ -3,6 +3,7 @@
 // @ts-check
 /// <reference path="../../core/lib.deno_core.d.ts" />
 /// <reference path="../webidl/internal.d.ts" />
+/// <reference path="../fetch/lib.deno_fetch.d.ts" />
 /// <reference path="../web/internal.d.ts" />
 /// <reference path="../web/lib.deno_web.d.ts" />
 /// <reference lib="esnext" />
@@ -203,6 +204,197 @@
     configurable: true,
   });
 
+  class TextDecoderStream {
+    /** @type {TextDecoder} */
+    #decoder;
+    /** @type {TransformStream<BufferSource, string>} */
+    #transform;
+
+    /**
+     *
+     * @param {string} label
+     * @param {TextDecoderOptions} options
+     */
+    constructor(label = "utf-8", options = {}) {
+      const prefix = "Failed to construct 'TextDecoderStream'";
+      label = webidl.converters.DOMString(label, {
+        prefix,
+        context: "Argument 1",
+      });
+      options = webidl.converters.TextDecoderOptions(options, {
+        prefix,
+        context: "Argument 2",
+      });
+      this.#decoder = new TextDecoder(label, options);
+      this.#transform = new TransformStream({
+        // The transform and flush functions need access to TextDecoderStream's
+        // `this`, so they are defined as functions rather than methods.
+        transform: (chunk, controller) => {
+          try {
+            chunk = webidl.converters.BufferSource(chunk, {
+              allowShared: true,
+            });
+            const decoded = this.#decoder.decode(chunk, { stream: true });
+            if (decoded) {
+              controller.enqueue(decoded);
+            }
+            return Promise.resolve();
+          } catch (err) {
+            return Promise.reject(err);
+          }
+        },
+        flush: (controller) => {
+          try {
+            const final = this.#decoder.decode();
+            if (final) {
+              controller.enqueue(final);
+            }
+            return Promise.resolve();
+          } catch (err) {
+            return Promise.reject(err);
+          }
+        },
+      });
+      this[webidl.brand] = webidl.brand;
+    }
+
+    /** @returns {string} */
+    get encoding() {
+      webidl.assertBranded(this, TextDecoderStream);
+      return this.#decoder.encoding;
+    }
+
+    /** @returns {boolean} */
+    get fatal() {
+      webidl.assertBranded(this, TextDecoderStream);
+      return this.#decoder.fatal;
+    }
+
+    /** @returns {boolean} */
+    get ignoreBOM() {
+      webidl.assertBranded(this, TextDecoderStream);
+      return this.#decoder.ignoreBOM;
+    }
+
+    /** @returns {ReadableStream<string>} */
+    get readable() {
+      webidl.assertBranded(this, TextDecoderStream);
+      return this.#transform.readable;
+    }
+
+    /** @returns {WritableStream<BufferSource>} */
+    get writable() {
+      webidl.assertBranded(this, TextDecoderStream);
+      return this.#transform.writable;
+    }
+
+    get [Symbol.toStringTag]() {
+      return "TextDecoderStream";
+    }
+  }
+
+  Object.defineProperty(TextDecoderStream.prototype, "encoding", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextDecoderStream.prototype, "fatal", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextDecoderStream.prototype, "ignoreBOM", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextDecoderStream.prototype, "readable", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextDecoderStream.prototype, "writable", {
+    enumerable: true,
+    configurable: true,
+  });
+
+  class TextEncoderStream {
+    /** @type {string | null} */
+    #pendingHighSurrogate = null;
+    /** @type {TransformStream<string, Uint8Array>} */
+    #transform;
+
+    constructor() {
+      this.#transform = new TransformStream({
+        // The transform and flush functions need access to TextEncoderStream's
+        // `this`, so they are defined as functions rather than methods.
+        transform: (chunk, controller) => {
+          try {
+            chunk = webidl.converters.DOMString(chunk);
+            if (this.#pendingHighSurrogate !== null) {
+              chunk = this.#pendingHighSurrogate + chunk;
+            }
+            const lastCodeUnit = chunk.charCodeAt(chunk.length - 1);
+            if (0xD800 <= lastCodeUnit && lastCodeUnit <= 0xDBFF) {
+              this.#pendingHighSurrogate = chunk.slice(-1);
+              chunk = chunk.slice(0, -1);
+            } else {
+              this.#pendingHighSurrogate = null;
+            }
+            if (chunk) {
+              controller.enqueue(core.encode(chunk));
+            }
+            return Promise.resolve();
+          } catch (err) {
+            return Promise.reject(err);
+          }
+        },
+        flush: (controller) => {
+          try {
+            if (this.#pendingHighSurrogate !== null) {
+              controller.enqueue(new Uint8Array([0xEF, 0xBF, 0xBD]));
+            }
+            return Promise.resolve();
+          } catch (err) {
+            return Promise.reject(err);
+          }
+        },
+      });
+      this[webidl.brand] = webidl.brand;
+    }
+
+    /** @returns {string} */
+    get encoding() {
+      webidl.assertBranded(this, TextEncoderStream);
+      return "utf-8";
+    }
+
+    /** @returns {ReadableStream<Uint8Array>} */
+    get readable() {
+      webidl.assertBranded(this, TextEncoderStream);
+      return this.#transform.readable;
+    }
+
+    /** @returns {WritableStream<string>} */
+    get writable() {
+      webidl.assertBranded(this, TextEncoderStream);
+      return this.#transform.writable;
+    }
+
+    get [Symbol.toStringTag]() {
+      return "TextEncoderStream";
+    }
+  }
+
+  Object.defineProperty(TextEncoderStream.prototype, "encoding", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextEncoderStream.prototype, "readable", {
+    enumerable: true,
+    configurable: true,
+  });
+  Object.defineProperty(TextEncoderStream.prototype, "writable", {
+    enumerable: true,
+    configurable: true,
+  });
+
   webidl.converters.TextDecoderOptions = webidl.createDictionaryConverter(
     "TextDecoderOptions",
     [
@@ -259,6 +451,8 @@
   window.__bootstrap.encoding = {
     TextEncoder,
     TextDecoder,
+    TextEncoderStream,
+    TextDecoderStream,
     decode,
   };
 })(this);

extensions/web/lib.deno_web.d.ts

@@ -195,8 +195,8 @@ declare class TextDecoder {
   readonly fatal: boolean;
   /** Returns `true` if ignore BOM flag is set, and `false` otherwise. */
   readonly ignoreBOM = false;
-  /** Returns the result of running encoding's decoder. */
 
+  /** Returns the result of running encoding's decoder. */
   decode(input?: BufferSource, options?: TextDecodeOptions): string;
 }
 
@@ -207,12 +207,33 @@ declare interface TextEncoderEncodeIntoResult {
 
 declare class TextEncoder {
   /** Returns "utf-8". */
-  readonly encoding = "utf-8";
+  readonly encoding: "utf-8";
   /** Returns the result of running UTF-8's encoder. */
   encode(input?: string): Uint8Array;
   encodeInto(input: string, dest: Uint8Array): TextEncoderEncodeIntoResult;
 }
 
+declare class TextDecoderStream {
+  /** Returns encoding's name, lowercased. */
+  readonly encoding: string;
+  /** Returns `true` if error mode is "fatal", and `false` otherwise. */
+  readonly fatal: boolean;
+  /** Returns `true` if ignore BOM flag is set, and `false` otherwise. */
+  readonly ignoreBOM = false;
+  constructor(label?: string, options?: TextDecoderOptions);
+  readonly readable: ReadableStream<string>;
+  readonly writable: WritableStream<BufferSource>;
+  readonly [Symbol.toStringTag]: string;
+}
+
+declare class TextEncoderStream {
+  /** Returns "utf-8". */
+  readonly encoding: "utf-8";
+  readonly readable: ReadableStream<Uint8Array>;
+  readonly writable: WritableStream<string>;
+  readonly [Symbol.toStringTag]: string;
+}
+
 /** A controller object that allows you to abort one or more DOM requests as and
  * when desired. */
 declare class AbortController {

runtime/js/99_main.js

@@ -287,6 +287,8 @@ delete Object.prototype.__proto__;
     Response: util.nonEnumerable(fetch.Response),
     TextDecoder: util.nonEnumerable(encoding.TextDecoder),
     TextEncoder: util.nonEnumerable(encoding.TextEncoder),
+    TextDecoderStream: util.nonEnumerable(encoding.TextDecoderStream),
+    TextEncoderStream: util.nonEnumerable(encoding.TextEncoderStream),
     TransformStream: util.nonEnumerable(streams.TransformStream),
     URL: util.nonEnumerable(url.URL),
     URLSearchParams: util.nonEnumerable(url.URLSearchParams),

tools/wpt/expectation.json

@@ -186,24 +186,7 @@
     "encodeInto.any.html": [
       "encodeInto() and a detached output buffer"
     ],
-    "idlharness.any.html": [
-      "TextDecoderStream interface: existence and properties of interface object",
-      "TextDecoderStream interface object length",
-      "TextDecoderStream interface object name",
-      "TextDecoderStream interface: existence and properties of interface prototype object",
-      "TextDecoderStream interface: existence and properties of interface prototype object's \"constructor\" property",
-      "TextDecoderStream interface: existence and properties of interface prototype object's @@unscopables property",
-      "TextDecoderStream interface: attribute encoding",
-      "TextDecoderStream interface: attribute fatal",
-      "TextDecoderStream interface: attribute ignoreBOM",
-      "TextEncoderStream interface: existence and properties of interface object",
-      "TextEncoderStream interface object length",
-      "TextEncoderStream interface object name",
-      "TextEncoderStream interface: existence and properties of interface prototype object",
-      "TextEncoderStream interface: existence and properties of interface prototype object's \"constructor\" property",
-      "TextEncoderStream interface: existence and properties of interface prototype object's @@unscopables property",
-      "TextEncoderStream interface: attribute encoding"
-    ],
+    "idlharness.any.html": true,
     "iso-2022-jp-decoder.any.html": true,
     "legacy-mb-schinese": {
       "gb18030": {
@@ -215,18 +198,20 @@
     },
     "replacement-encodings.any.html": false,
     "streams": {
-      "backpressure.any.html": false,
-      "decode-attributes.any.html": false,
-      "decode-bad-chunks.any.html": false,
-      "decode-ignore-bom.any.html": false,
-      "decode-incomplete-input.any.html": false,
-      "decode-non-utf8.any.html": false,
-      "decode-split-character.any.html": false,
-      "decode-utf8.any.html": false,
-      "encode-bad-chunks.any.html": false,
-      "encode-utf8.any.html": false,
-      "readable-writable-properties.any.html": false,
-      "realms.window.html": false
+      "backpressure.any.html": true,
+      "decode-attributes.any.html": true,
+      "decode-bad-chunks.any.html": true,
+      "decode-ignore-bom.any.html": true,
+      "decode-incomplete-input.any.html": true,
+      "decode-non-utf8.any.html": true,
+      "decode-split-character.any.html": true,
+      "decode-utf8.any.html": [
+        "decoding a transferred Uint8Array chunk should give no output",
+        "decoding a transferred ArrayBuffer chunk should give no output"
+      ],
+      "encode-bad-chunks.any.html": true,
+      "encode-utf8.any.html": true,
+      "readable-writable-properties.any.html": true
     },
     "textdecoder-arguments.any.html": true,
     "textdecoder-byte-order-marks.any.html": true,