[io] Make it possible to change the line ending output by stdout and stderr.

There is a performance impact in:
`stdout.lineTerminator = "\r\n";`

For small writes (<100 chars), the performance loss is lost in the noise of the `write` system call.

For writes of ~500 chars, the performance is about half of that without line terminator translation. But, on a M2 Mac laptop, ~80M characters can be written per second.

Bug: https://github.com/dart-lang/sdk/issues/53161
Change-Id: Icfa0f981dcf6edb856d8aac5e0e270bc0148d498
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/326761
Reviewed-by: Siva Annamalai <asiva@google.com>
Reviewed-by: Lasse Nielsen <lrn@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Reviewed-by: Ömer Ağacan <omersa@google.com>
Reviewed-by: Brian Quinlan <bquinlan@google.com>
Commit-Queue: Brian Quinlan <bquinlan@google.com>

CHANGELOG.md

@@ -17,6 +17,17 @@
 [#54640]: https://github.com/dart-lang/sdk/issues/54640
 [#54828]: https://github.com/dart-lang/sdk/issues/54828
 
+### Libraries
+
+#### `dart:io`
+
+- **Breaking change** [#53863][]: `Stdout` has a new field `lineTerminator`,
+  which allows developers to control the line ending used by `stdout` and
+  `stderr`. Classes that `implement Stdout` must define the `lineTerminator`
+  field. The default semantics of `stdout` and `stderr` are not changed.
+
+[#53863]: https://github.com/dart-lang/sdk/issues/53863
+
 ### Tools
 
 #### Pub

sdk/lib/io/stdio.dart

@@ -212,6 +212,10 @@ class Stdin extends _StdStream implements Stream<List<int>> {
 /// The [addError] API is inherited from [StreamSink] and calling it will result
 /// in an unhandled asynchronous error unless there is an error handler on
 /// [done].
+///
+/// The [lineTerminator] field is used by the [write], [writeln], [writeAll]
+/// and [writeCharCode] methods to translate `"\n"`. By default, `"\n"` is
+/// output literally.
 class Stdout extends _StdSink implements IOSink {
   final int _fd;
   IOSink? _nonBlocking;
@@ -261,6 +265,9 @@ class Stdout extends _StdSink implements IOSink {
   external static bool _supportsAnsiEscapes(int fd);
 
   /// A non-blocking `IOSink` for the same output.
+  ///
+  /// The returned `IOSink` will be initialized with an [encoding] of UTF-8 and
+  /// will not do line ending conversion.
   IOSink get nonBlocking {
     return _nonBlocking ??= new IOSink(new _FileStreamConsumer.fromStdio(_fd));
   }
@@ -324,29 +331,114 @@ class _StdConsumer implements StreamConsumer<List<int>> {
   }
 }
 
+/// Pattern matching a "\n" character not following a "\r".
+///
+/// Used to replace such with "\r\n" in the [_StdSink] write methods.
+final _newLineDetector = RegExp(r'(?<!\r)\n');
+
+/// Pattern matching "\n" characters not following a "\r", or at the start of
+/// input.
+///
+/// Used to replace those with "\r\n" in the [_StdSink] write methods,
+/// when the previously written string ended in a \r character.
+final _newLineDetectorAfterCr = RegExp(r'(?<!\r|^)\n');
+
 class _StdSink implements IOSink {
   final IOSink _sink;
+  bool _windowsLineTerminator = false;
+  bool _lastWrittenCharIsCR = false;
 
   _StdSink(this._sink);
 
+  /// Line ending appended by [writeln], and replacing `"\n"` in some methods.
+  ///
+  /// Must be one of the values `"\n"` (the default) or `"\r\n"`.
+  ///
+  /// When set to `"\r\n"`, the methods [write], [writeln], [writeAll] and
+  /// [writeCharCode] will convert embedded newlines, `"\n"`, in their
+  /// arguments to `"\r\n"`. If their arguments already contain `"\r\n"`
+  /// sequences, then these sequences will be not be converted. This is true
+  /// even if the sequence is generated across different method calls.
+  ///
+  /// If `lineTerminator` is `"\n"` then the written strings are not modified.
+  //
+  /// Setting `lineTerminator` to [Platform.lineTerminator] will result in
+  /// "write" methods outputting the line endings for the platform:
+  ///
+  /// ```dart
+  /// stdout.lineTerminator = Platform.lineTerminator;
+  /// stderr.lineTerminator = Platform.lineTerminator;
+  /// ```
+  ///
+  /// The value of `lineTerminator` has no effect on byte-oriented methods
+  /// such as [add].
+  ///
+  /// The value of `lineTerminator` does not effect the output of the [print]
+  /// function.
+  ///
+  /// Throws [ArgumentError] if set to a value other than `"\n"` or `"\r\n"`.
+  String get lineTerminator => _windowsLineTerminator ? "\r\n" : "\n";
+  set lineTerminator(String lineTerminator) {
+    if (lineTerminator == "\r\n") {
+      assert(!_lastWrittenCharIsCR || _windowsLineTerminator);
+      _windowsLineTerminator = true;
+    } else if (lineTerminator == "\n") {
+      _windowsLineTerminator = false;
+      _lastWrittenCharIsCR = false;
+    } else {
+      throw ArgumentError.value(lineTerminator, "lineTerminator",
+          r'invalid line terminator, must be one of "\r" or "\r\n"');
+    }
+  }
+
   Encoding get encoding => _sink.encoding;
   void set encoding(Encoding encoding) {
     _sink.encoding = encoding;
   }
 
-  void write(Object? object) {
-    _sink.write(object);
+  void _write(Object? object) {
+    if (!_windowsLineTerminator) {
+      _sink.write(object);
+      return;
+    }
+
+    var string = '$object';
+    if (string.isEmpty) return;
+    if (_lastWrittenCharIsCR) {
+      string = string.replaceAll(_newLineDetectorAfterCr, "\r\n");
+    } else {
+      string = string.replaceAll(_newLineDetector, "\r\n");
+    }
+    _lastWrittenCharIsCR = string.endsWith('\r');
+    _sink.write(string);
   }
 
+  void write(Object? object) => _write(object);
+
   void writeln([Object? object = ""]) {
-    _sink.writeln(object);
+    _write(object);
+    _sink.write(_windowsLineTerminator ? "\r\n" : "\n");
+    _lastWrittenCharIsCR = false;
   }
 
   void writeAll(Iterable objects, [String sep = ""]) {
-    _sink.writeAll(objects, sep);
+    Iterator iterator = objects.iterator;
+    if (!iterator.moveNext()) return;
+    if (sep.isEmpty) {
+      do {
+        _write(iterator.current);
+      } while (iterator.moveNext());
+    } else {
+      _write(iterator.current);
+      while (iterator.moveNext()) {
+        _write(sep);
+        _write(iterator.current);
+      }
+    }
   }
 
   void add(List<int> data) {
+    _lastWrittenCharIsCR = false;
     _sink.add(data);
   }
 
@@ -355,10 +447,19 @@ class _StdSink implements IOSink {
   }
 
   void writeCharCode(int charCode) {
-    _sink.writeCharCode(charCode);
+    if (!_windowsLineTerminator) {
+      _sink.writeCharCode(charCode);
+      return;
+    }
+
+    _write(String.fromCharCode(charCode));
+  }
+
+  Future addStream(Stream<List<int>> stream) {
+    _lastWrittenCharIsCR = false;
+    return _sink.addStream(stream);
   }
 
-  Future addStream(Stream<List<int>> stream) => _sink.addStream(stream);
   Future flush() => _sink.flush();
   Future close() => _sink.close();
   Future get done => _sink.done;

tests/standalone/io/stdout_stderr_test.dart

@@ -2,43 +2,207 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// OtherResources=stdout_stderr_test_script.dart
+
 import "package:expect/expect.dart";
 import "dart:async";
 import "dart:convert";
 import "dart:io";
 
-callIOSink(IOSink sink) {
-  // Call all methods on IOSink.
-  sink.encoding = ascii;
-  Expect.equals(ascii, sink.encoding);
-  sink.write("Hello\n");
-  sink.writeln("Hello");
-  sink.writeAll(["H", "e", "l", "lo\n"]);
-  sink.writeCharCode(72);
-  sink.add([101, 108, 108, 111, 10]);
+/// Execute "stdout_stderr_test_script.dart" with `command` as an argument and
+/// return the commands stdout as a list of bytes.
+List<int> runTest(String lineTerminatorMode, String encoding, String command) {
+  final result = Process.runSync(
+      Platform.executable,
+      []
+        ..addAll(Platform.executableArguments)
+        ..add('--verbosity=warning')
+        ..add(Platform.script
+            .resolve('stdout_stderr_test_script.dart')
+            .toFilePath())
+        ..add('--eol=$lineTerminatorMode')
+        ..add('--encoding=$encoding')
+        ..add(command),
+      stdoutEncoding: null);
 
-  var controller = new StreamController<List<int>>(sync: true);
-  var future = sink.addStream(controller.stream);
-  controller.add([72, 101, 108]);
-  controller.add([108, 111, 10]);
-  controller.close();
-
-  future.then((_) {
-    controller = new StreamController<List<int>>(sync: true);
-    controller.stream.pipe(sink);
-    controller.add([72, 101, 108]);
-    controller.add([108, 111, 10]);
-    controller.close();
-  });
+  if (result.exitCode != 0) {
+    throw AssertionError(
+        'unexpected exit code for command $command: ${result.stderr}');
+  }
+  return result.stdout;
 }
 
-main() {
-  callIOSink(stdout);
-  stdout.done.then((_) {
-    callIOSink(stderr);
-    stderr.done.then((_) {
-      stdout.close();
-      stderr.close();
-    });
-  });
+const winEol = [13, 10];
+const posixEol = [10];
+
+void testByteListHello() {
+  // add([104, 101, 108, 108, 111, 10])
+  final expected = [104, 101, 108, 108, 111, 10];
+  Expect.listEquals(expected, runTest("unix", "ascii", "byte-list-hello"));
+  Expect.listEquals(expected, runTest("windows", "ascii", "byte-list-hello"));
+  Expect.listEquals(expected, runTest("default", "ascii", "byte-list-hello"));
+}
+
+void testByteListAllo() {
+  // add([97, 108, 108, 244, 10])
+  final expected = [97, 108, 108, 244, 10];
+  Expect.listEquals(expected, runTest("unix", "latin1", "byte-list-allo"));
+  Expect.listEquals(expected, runTest("windows", "latin1", "byte-list-allo"));
+  Expect.listEquals(expected, runTest("default", "latin1", "byte-list-allo"));
+}
+
+void testStreamHello() {
+  // add([104, 101, 108, 108, 111, 10])
+  final expected = [104, 101, 108, 108, 111, 10];
+  Expect.listEquals(expected, runTest("unix", "ascii", "stream-hello"));
+  Expect.listEquals(expected, runTest("windows", "ascii", "stream-hello"));
+  Expect.listEquals(expected, runTest("default", "ascii", "stream-hello"));
+}
+
+void testStreamAllo() {
+  // add([97, 108, 108, 244, 10])
+  final expected = [97, 108, 108, 244, 10];
+  Expect.listEquals(expected, runTest("unix", "latin1", "stream-allo"));
+  Expect.listEquals(expected, runTest("windows", "latin1", "stream-allo"));
+  Expect.listEquals(expected, runTest("default", "latin1", "stream-allo"));
+}
+
+void testStringHello() {
+  // write('hello\n')
+  final expectedPosix = [104, 101, 108, 108, 111, ...posixEol];
+  final expectedWin = [104, 101, 108, 108, 111, ...winEol];
+
+  Expect.listEquals(expectedPosix, runTest("unix", "ascii", "string-hello"));
+  Expect.listEquals(expectedWin, runTest("windows", "ascii", "string-hello"));
+  Expect.listEquals(expectedPosix, runTest("default", "ascii", "string-hello"));
+}
+
+void testStringAllo() {
+  // write('hello\n')
+  final expectedPosix = [97, 108, 108, 244, ...posixEol];
+  final expectedWin = [97, 108, 108, 244, ...winEol];
+
+  Expect.listEquals(expectedPosix, runTest("unix", "ascii", "string-allo"));
+  Expect.listEquals(expectedWin, runTest("windows", "ascii", "string-allo"));
+  Expect.listEquals(expectedPosix, runTest("default", "ascii", "string-allo"));
+}
+
+void testStringInternalLineFeeds() {
+  // write('l1\nl2\nl3')
+  final expectedPosix = [108, 49, ...posixEol, 108, 50, ...posixEol, 108, 51];
+  final expectedWin = [108, 49, ...winEol, 108, 50, ...winEol, 108, 51];
+
+  Expect.listEquals(
+      expectedPosix, runTest("unix", "ascii", "string-internal-linefeeds"));
+  Expect.listEquals(
+      expectedWin, runTest("windows", "ascii", "string-internal-linefeeds"));
+  Expect.listEquals(
+      expectedPosix, runTest("default", "ascii", "string-internal-linefeeds"));
+}
+
+void testStringCarriageReturns() {
+  // write("l1\rl2\rl3\r")
+  final expected = [108, 49, 13, 108, 50, 13, 108, 51, 13];
+  Expect.listEquals(
+      expected, runTest("unix", "ascii", "string-internal-carriagereturns"));
+  Expect.listEquals(
+      expected, runTest("windows", "ascii", "string-internal-carriagereturns"));
+  Expect.listEquals(
+      expected, runTest("default", "ascii", "string-internal-carriagereturns"));
+}
+
+void testStringCarriageReturnLinefeeds() {
+  // ""l1\r\nl2\r\nl3\r\n""
+  final expected = [108, 49, ...winEol, 108, 50, ...winEol, 108, 51, ...winEol];
+  Expect.listEquals(expected,
+      runTest("unix", "ascii", "string-internal-carriagereturn-linefeeds"));
+  Expect.listEquals(expected,
+      runTest("windows", "ascii", "string-internal-carriagereturn-linefeeds"));
+  Expect.listEquals(expected,
+      runTest("default", "ascii", "string-internal-carriagereturn-linefeeds"));
+}
+
+void testStringCarriageReturnLinefeedsSeperateWrite() {
+  // write("l1\r");
+  // write("\nl2");
+  final expected = [108, 49, ...winEol, 108, 50];
+  Expect.listEquals(
+      expected,
+      runTest(
+          "unix", "ascii", "string-carriagereturn-linefeed-seperate-write"));
+  Expect.listEquals(
+      expected,
+      runTest(
+          "windows", "ascii", "string-carriagereturn-linefeed-seperate-write"));
+  Expect.listEquals(
+      expected,
+      runTest(
+          "default", "ascii", "string-carriagereturn-linefeed-seperate-write"));
+}
+
+void testStringCarriageReturnFollowedByWriteln() {
+  // write("l1\r");
+  // writeln();
+  final expectedPosix = [108, 49, 13, ...posixEol];
+  final expectedWin = [108, 49, 13, ...winEol];
+
+  Expect.listEquals(
+      expectedPosix, runTest("unix", "ascii", "string-carriagereturn-writeln"));
+  Expect.listEquals(expectedWin,
+      runTest("windows", "ascii", "string-carriagereturn-writeln"));
+  Expect.listEquals(expectedPosix,
+      runTest("default", "ascii", "string-carriagereturn-writeln"));
+}
+
+void testWriteCharCodeLineFeed() {
+  // write("l1");
+  // writeCharCode(10);
+  final expectedPosix = [108, 49, ...posixEol];
+  final expectedWin = [108, 49, ...winEol];
+
+  Expect.listEquals(
+      expectedPosix, runTest("unix", "ascii", "write-char-code-linefeed"));
+  Expect.listEquals(
+      expectedWin, runTest("windows", "ascii", "write-char-code-linefeed"));
+  Expect.listEquals(
+      expectedPosix, runTest("default", "ascii", "write-char-code-linefeed"));
+}
+
+void testWriteCharCodeLineFeedFollowingCarriageReturn() {
+  // write("1\r");
+  // writeCharCode(10);
+  final expected = [108, 49, ...winEol];
+
+  Expect.listEquals(
+      expected,
+      runTest(
+          "unix", "ascii", "write-char-code-linefeed-after-carriagereturn"));
+  Expect.listEquals(
+      expected,
+      runTest(
+          "windows", "ascii", "write-char-code-linefeed-after-carriagereturn"));
+  Expect.listEquals(
+      expected,
+      runTest(
+          "default", "ascii", "write-char-code-linefeed-after-carriagereturn"));
+}
+
+void testInvalidLineTerminator() {
+  Expect.throwsArgumentError(() => stdout.lineTerminator = "\r");
+}
+
+void main() {
+  testByteListHello();
+  testByteListAllo();
+  testStreamHello();
+  testStreamAllo();
+  testStringHello();
+  testStringInternalLineFeeds();
+  testStringCarriageReturns();
+  testStringCarriageReturnLinefeeds();
+  testStringCarriageReturnLinefeedsSeperateWrite();
+  testStringCarriageReturnFollowedByWriteln();
+  testWriteCharCodeLineFeed();
+  testWriteCharCodeLineFeedFollowingCarriageReturn();
+  testInvalidLineTerminator();
 }

tests/standalone/io/stdout_stderr_test_script.dart

@@ -0,0 +1,101 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// This is a companion script to print_test.dart.
+
+import 'dart:convert';
+import 'dart:io';
+import 'dart:async';
+
+class ToString {
+  String _toString;
+
+  ToString(this._toString);
+
+  String toString() => _toString;
+}
+
+main(List<String> arguments) {
+  switch (arguments[0]) {
+    case "--eol=default":
+      break;
+    case "--eol=windows":
+      stdout.lineTerminator = '\r\n';
+      break;
+    case "--eol=unix":
+      stdout.lineTerminator = '\n';
+      break;
+    default:
+      stderr.writeln("eol mode not recognized: ${arguments[0]}");
+      exit(1);
+      break;
+  }
+
+  if (!arguments[1].startsWith("--encoding=")) {
+    stderr.writeln("encoding not recognized: ${arguments[0]}");
+    exit(1);
+  }
+
+  stdout.encoding =
+      Encoding.getByName(arguments[1].replaceFirst("--encoding=", ""))!;
+
+  switch (arguments.last) {
+    case "byte-list-hello":
+      stdout.add([104, 101, 108, 108, 111, 10]);
+      break;
+    case "byte-list-allo":
+      stdout.add([97, 108, 108, 244, 10]);
+      break;
+    case "stream-hello":
+      var controller = new StreamController<List<int>>(sync: true);
+      stdout.addStream(controller.stream);
+      controller.add([104, 101, 108, 108]);
+      controller.add([111, 10]);
+      controller.close();
+      break;
+    case "stream-allo":
+      var controller = new StreamController<List<int>>(sync: true);
+      stdout.addStream(controller.stream);
+      controller.add([97, 108, 108]);
+      controller.add([244, 10]);
+      controller.close();
+      break;
+    case "string-hello":
+      stdout.write('hello\n');
+      break;
+    case "string-allo":
+      stdout.write('allô\n');
+      break;
+    case "string-internal-linefeeds":
+      stdout.write("l1\nl2\nl3");
+      break;
+    case "string-internal-carriagereturns":
+      stdout.write("l1\rl2\rl3\r");
+      break;
+    case "string-internal-carriagereturn-linefeeds":
+      stdout.write("l1\r\nl2\r\nl3\r\n");
+      break;
+    case "string-carriagereturn-linefeed-seperate-write":
+      stdout.write("l1\r");
+      stdout.write("\nl2");
+      break;
+    case "string-carriagereturn-writeln":
+      stdout.write("l1\r");
+      stdout.writeln();
+      break;
+    case "write-char-code-linefeed":
+      stdout.write("l1");
+      stdout.writeCharCode(10);
+    case "write-char-code-linefeed-after-carriagereturn":
+      stdout.write("l1\r");
+      stdout.writeCharCode(10);
+    case "object-internal-linefeeds":
+      print(ToString("l1\nl2\nl3"));
+      break;
+    default:
+      stderr.writeln("Command was not recognized");
+      exit(1);
+      break;
+  }
+}

tests/standalone/standalone_precompiled.status

@@ -45,6 +45,7 @@ io/signals_test: Skip
 io/stdin_sync_test: Skip
 io/stdio_implicit_close_test: Skip
 io/stdio_nonblocking_test: Skip
+io/stdout_stderr_test: Skip
 io/test_extension_fail_test: Skip
 io/test_extension_test: Skip
 io/windows_environment_test: Skip