[vm/corelib] Fix int/double comparisons in num.compareTo

* double.compareTo(int) is fixed to handle integers which are not
  precisely representable as doubles.

* int.compareTo(double) is fixed to properly handle doubles which
  are clamped when converted to integers (with Dart 2.0 fixed-size
  integers).

* Test for num.compareTo is updated for fixed-size integers; more corner
  cases added.

Closes https://github.com/dart-lang/sdk/issues/31917

Change-Id: I8e98c3627f032082ab6a397713dece436585afd1
Reviewed-on: https://dart-review.googlesource.com/35004
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Lasse R.H. Nielsen <lrn@google.com>
Reviewed-by: Régis Crelier <regis@google.com>

runtime/lib/double.dart

@@ -288,6 +288,30 @@ class _Double implements double {
           return EQUAL;
         }
         return thisIsNegative ? LESS : GREATER;
+      } else if (other is int) {
+        // Compare as integers as it is more precise if the integer value is
+        // outside of MIN_EXACT_INT_TO_DOUBLE..MAX_EXACT_INT_TO_DOUBLE range.
+        const int MAX_EXACT_INT_TO_DOUBLE = 9007199254740992; // 2^53.
+        const int MIN_EXACT_INT_TO_DOUBLE = -MAX_EXACT_INT_TO_DOUBLE;
+        if ((MIN_EXACT_INT_TO_DOUBLE <= other) &&
+            (other <= MAX_EXACT_INT_TO_DOUBLE)) {
+          return EQUAL;
+        }
+        const bool limitIntsTo64Bits = ((1 << 64) == 0);
+        if (limitIntsTo64Bits) {
+          // With integers limited to 64 bits, double.toInt() clamps
+          // double value to fit into the MIN_INT64..MAX_INT64 range.
+          // MAX_INT64 is not precisely representable as double, so
+          // integers near MAX_INT64 compare as equal to (MAX_INT64 + 1) when
+          // represented as doubles.
+          // There is no similar problem with MIN_INT64 as it is precisely
+          // representable as double.
+          const double maxInt64Plus1AsDouble = 9223372036854775808.0;
+          if (this >= maxInt64Plus1AsDouble) {
+            return GREATER;
+          }
+        }
+        return toInt().compareTo(other);
       } else {
         return EQUAL;
       }

runtime/lib/integers.dart

@@ -166,20 +166,36 @@ abstract class _IntegerImplementation implements int {
     if (other is double) {
       const int MAX_EXACT_INT_TO_DOUBLE = 9007199254740992; // 2^53.
       const int MIN_EXACT_INT_TO_DOUBLE = -MAX_EXACT_INT_TO_DOUBLE;
-      double d = other;
-      if (d.isInfinite) {
-        return d == double.negativeInfinity ? GREATER : LESS;
+      const bool limitIntsTo64Bits = ((1 << 64) == 0);
+      if (limitIntsTo64Bits) {
+        // With integers limited to 64 bits, double.toInt() clamps
+        // double value to fit into the MIN_INT64..MAX_INT64 range.
+        // Check if the double value is outside of this range.
+        // This check handles +/-infinity as well.
+        const double minInt64AsDouble = -9223372036854775808.0;
+        // MAX_INT64 is not precisely representable in doubles, so
+        // check against (MAX_INT64 + 1).
+        const double maxInt64Plus1AsDouble = 9223372036854775808.0;
+        if (other < minInt64AsDouble) {
+          return GREATER;
+        } else if (other >= maxInt64Plus1AsDouble) {
+          return LESS;
+        }
+      } else {
+        if (other.isInfinite) {
+          return other.isNegative ? GREATER : LESS;
+        }
       }
-      if (d.isNaN) {
+      if (other.isNaN) {
         return LESS;
       }
       if (MIN_EXACT_INT_TO_DOUBLE <= this && this <= MAX_EXACT_INT_TO_DOUBLE) {
         // Let the double implementation deal with -0.0.
-        return -(d.compareTo(this.toDouble()));
+        return -(other.compareTo(this.toDouble()));
       } else {
         // If abs(other) > MAX_EXACT_INT_TO_DOUBLE, then other has an integer
         // value (no bits below the decimal point).
-        other = d.toInt();
+        other = other.toInt();
       }
     }
     if (this < other) {

tests/corelib_2/compare_to2_test.dart

@@ -23,16 +23,18 @@ main() {
   var two53p1 = two53 + 1;
   var maxFiniteAsDouble = 1.7976931348623157e+308;
   var maxFiniteAsInt = maxFiniteAsDouble.truncate();
-  int huge = 1 << 2000;
-  int hugeP1 = huge + 1;
   var inf = double.infinity;
   var nan = double.nan;
   var mnan = negate(nan);
+  var minInt64 = -0x8000000000000000;
+  var minInt64AsDouble = minInt64.toDouble();
+  var maxInt64 = 0x7fffffffffffffff;
+  var maxInt64AsDouble = maxInt64.toDouble(); // 1 << 63
   var matrix = [
     -inf,
-    -hugeP1,
-    -huge,
-    [-maxFiniteAsDouble, -maxFiniteAsInt],
+    -maxFiniteAsDouble,
+    [minInt64, minInt64AsDouble],
+    [-maxInt64, -maxFiniteAsInt],
     -two53p1,
     [-two53, -maxExactIntAsInt, -maxExactIntAsDouble],
     -maxNonInt,
@@ -57,9 +59,9 @@ main() {
     maxNonInt,
     [two53, maxExactIntAsInt, maxExactIntAsDouble],
     two53p1,
-    [maxFiniteAsDouble, maxFiniteAsInt],
-    huge,
-    hugeP1,
+    [maxInt64, maxFiniteAsInt],
+    maxInt64AsDouble,
+    maxFiniteAsDouble,
     inf,
     [nan, mnan],
   ];

tests/corelib_2/corelib_2.status

@@ -330,7 +330,6 @@ bigint_test: Pass, Timeout # Please triage.
 [ $compiler == dartk && $runtime == vm && $strong ]
 apply3_test: CompileTimeError # Issue 31402 (Invocation arguments)
 bool_from_environment2_test/03: MissingCompileTimeError
-compare_to2_test: RuntimeError
 iterable_fold_test/02: RuntimeError
 iterable_reduce_test/01: CompileTimeError # Issue 31533
 iterable_reduce_test/none: RuntimeError
@@ -357,7 +356,6 @@ unicode_test: RuntimeError # Issue 18061: German double S.
 # ===== dartkp + dart_precompiled status lines =====
 [ $compiler == dartkp && $runtime == dart_precompiled && $strong ]
 bool_from_environment2_test/03: MissingCompileTimeError
-compare_to2_test: RuntimeError
 iterable_fold_test/02: RuntimeError
 iterable_reduce_test/01: CompileTimeError # Issue 31533
 iterable_reduce_test/none: RuntimeError
@@ -556,7 +554,6 @@ regexp/stack-overflow_test: RuntimeError, OK # Smaller limit with irregex interp
 int_parse_radix_bad_handler_test: MissingCompileTimeError
 
 [ $compiler == app_jit || $compiler == none || $compiler == precompiler ]
-compare_to2_test: Fail # Issue 4018
 symbol_operator_test/03: Fail # Issue 11669
 symbol_reserved_word_test/06: RuntimeError # Issue 11669, With the exception of 'void', new Symbol() should not accept reserved words.
 symbol_reserved_word_test/09: RuntimeError # Issue 11669, With the exception of 'void', new Symbol() should not accept reserved words.