[vm, compiler] Load unboxed doubles directly from the literal pool.

TEST=ci
Change-Id: I4f881bc3e29059bce09eb0aa0ba99928e7fb10a8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/244120
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -1541,10 +1541,6 @@ void Assembler::Drop(intptr_t stack_elements) {
   }
 }
 
-intptr_t Assembler::FindImmediate(int32_t imm) {
-  return object_pool_builder().FindImmediate(imm);
-}
-
 // Uses a code sequence that can easily be decoded.
 void Assembler::LoadWordFromPoolIndex(Register rd,
                                       intptr_t index,
@@ -2840,7 +2836,15 @@ void Assembler::LoadDImmediate(DRegister dd,
                                Condition cond) {
   ASSERT(scratch != PC);
   ASSERT(scratch != IP);
-  if (!vmovd(dd, value, cond)) {
+  if (vmovd(dd, value, cond)) return;
+
+  int64_t imm64 = bit_cast<int64_t, double>(value);
+  if (constant_pool_allowed()) {
+    intptr_t index = object_pool_builder().FindImmediate64(imm64);
+    intptr_t offset =
+        target::ObjectPool::element_offset(index) - kHeapObjectTag;
+    LoadDFromOffset(dd, PP, offset, cond);
+  } else {
     // A scratch register and IP are needed to load an arbitrary double.
     ASSERT(scratch != kNoRegister);
     int64_t imm64 = bit_cast<int64_t, double>(value);
@@ -2850,6 +2854,13 @@ void Assembler::LoadDImmediate(DRegister dd,
   }
 }
 
+void Assembler::LoadQImmediate(QRegister qd, simd128_value_t value) {
+  ASSERT(constant_pool_allowed());
+  intptr_t index = object_pool_builder().FindImmediate128(value);
+  intptr_t offset = target::ObjectPool::element_offset(index) - kHeapObjectTag;
+  LoadMultipleDFromOffset(EvenDRegisterOf(qd), 2, PP, offset);
+}
+
 void Assembler::LoadFromOffset(Register reg,
                                const Address& address,
                                OperandSize size,

runtime/vm/compiler/assembler/assembler_arm.h

@@ -874,6 +874,7 @@ class Assembler : public AssemblerBase {
                       double value,
                       Register scratch,
                       Condition cond = AL);
+  void LoadQImmediate(QRegister dd, simd128_value_t value);
 
   void MarkExceptionHandler(Label* label);
 
@@ -985,7 +986,6 @@ class Assembler : public AssemblerBase {
                                Register scratch,
                                bool can_be_null = false) override;
 
-  intptr_t FindImmediate(int32_t imm);
   bool CanLoadFromObjectPool(const Object& object) const;
   void LoadFromOffset(Register reg,
                       const Address& address,

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -476,10 +476,6 @@ void Assembler::LoadDoubleWordFromPoolIndex(Register lower,
   }
 }
 
-intptr_t Assembler::FindImmediate(int64_t imm) {
-  return object_pool_builder().FindImmediate(imm);
-}
-
 bool Assembler::CanLoadFromObjectPool(const Object& object) const {
   ASSERT(IsOriginalObject(object));
   if (!constant_pool_allowed()) {
@@ -643,7 +639,7 @@ void Assembler::LoadImmediate(Register reg, int64_t imm) {
 
   // Use constant pool if allowed, unless we can load imm with 2 instructions.
   if ((w1 != 0) && constant_pool_allowed()) {
-    const intptr_t index = FindImmediate(imm);
+    const intptr_t index = object_pool_builder().FindImmediate(imm);
     LoadWordFromPoolIndex(reg, index);
     return;
   }
@@ -679,13 +675,26 @@ void Assembler::LoadImmediate(Register reg, int64_t imm) {
 }
 
 void Assembler::LoadDImmediate(VRegister vd, double immd) {
-  if (!fmovdi(vd, immd)) {
-    int64_t imm = bit_cast<int64_t, double>(immd);
-    LoadImmediate(TMP, imm);
+  if (fmovdi(vd, immd)) return;
+
+  int64_t imm64 = bit_cast<int64_t, double>(immd);
+  if (constant_pool_allowed()) {
+    intptr_t index = object_pool_builder().FindImmediate64(imm64);
+    intptr_t offset = target::ObjectPool::element_offset(index);
+    LoadDFromOffset(vd, PP, offset);
+  } else {
+    LoadImmediate(TMP, imm64);
     fmovdr(vd, TMP);
   }
 }
 
+void Assembler::LoadQImmediate(VRegister vd, simd128_value_t immq) {
+  ASSERT(constant_pool_allowed());
+  intptr_t index = object_pool_builder().FindImmediate128(immq);
+  intptr_t offset = target::ObjectPool::element_offset(index);
+  LoadQFromOffset(vd, PP, offset);
+}
+
 void Assembler::Branch(const Code& target,
                        Register pp,
                        ObjectPoolBuilderEntry::Patchability patchable) {

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -2041,7 +2041,6 @@ class Assembler : public AssemblerBase {
   compiler::LRState lr_state() const { return lr_state_; }
   void set_lr_state(compiler::LRState state) { lr_state_ = state; }
 
-  intptr_t FindImmediate(int64_t imm);
   bool CanLoadFromObjectPool(const Object& object) const;
   void LoadNativeEntry(Register dst,
                        const ExternalLabel* label,
@@ -2057,6 +2056,7 @@ class Assembler : public AssemblerBase {
   void LoadImmediate(Register reg, int64_t imm);
 
   void LoadDImmediate(VRegister reg, double immd);
+  void LoadQImmediate(VRegister reg, simd128_value_t immq);
 
   // Load word from pool from the given offset using encoding that
   // InstructionPattern::DecodeLoadWordFromPool can decode.

runtime/vm/compiler/assembler/assembler_base.cc

@@ -309,11 +309,25 @@ void AssemblerBase::Stop(const char* message) {
 }
 
 uword ObjIndexPair::Hash(Key key) {
-  if (key.type() != ObjectPoolBuilderEntry::kTaggedObject) {
-    return key.raw_value_;
+  switch (key.type()) {
+    case ObjectPoolBuilderEntry::kImmediate128:
+      return key.imm128_.int_storage[0] ^ key.imm128_.int_storage[1] ^
+             key.imm128_.int_storage[2] ^ key.imm128_.int_storage[3];
+
+#if defined(TARGET_ARCH_IS_32_BIT)
+    case ObjectPoolBuilderEntry::kImmediate64:
+      return key.imm64_;
+#endif
+    case ObjectPoolBuilderEntry::kImmediate:
+    case ObjectPoolBuilderEntry::kNativeFunction:
+    case ObjectPoolBuilderEntry::kSwitchableCallMissEntryPoint:
+    case ObjectPoolBuilderEntry::kMegamorphicCallEntryPoint:
+      return key.imm_;
+    case ObjectPoolBuilderEntry::kTaggedObject:
+      return ObjectHash(*key.obj_);
   }
 
-  return ObjectHash(*key.obj_);
+  UNREACHABLE();
 }
 
 void ObjectPoolBuilder::Reset() {
@@ -342,6 +356,22 @@ intptr_t ObjectPoolBuilder::AddImmediate(uword imm) {
                              ObjectPoolBuilderEntry::kNotPatchable));
 }
 
+intptr_t ObjectPoolBuilder::AddImmediate64(uint64_t imm) {
+#if defined(TARGET_ARCH_IS_32_BIT)
+  return AddObject(
+      ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate64,
+                             ObjectPoolBuilderEntry::kNotPatchable));
+#else
+  return AddImmediate(imm);
+#endif
+}
+
+intptr_t ObjectPoolBuilder::AddImmediate128(simd128_value_t imm) {
+  return AddObject(
+      ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate128,
+                             ObjectPoolBuilderEntry::kNotPatchable));
+}
+
 intptr_t ObjectPoolBuilder::AddObject(ObjectPoolBuilderEntry entry) {
   ASSERT((entry.type() != ObjectPoolBuilderEntry::kTaggedObject) ||
          (IsNotTemporaryScopedHandle(*entry.obj_) &&
@@ -359,6 +389,38 @@ intptr_t ObjectPoolBuilder::AddObject(ObjectPoolBuilderEntry entry) {
     }
   }
 
+#if defined(TARGET_ARCH_IS_32_BIT)
+  if (entry.type() == ObjectPoolBuilderEntry::kImmediate64) {
+    ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
+    uint64_t imm = entry.imm64_;
+    intptr_t idx = AddImmediate(Utils::Low32Bits(imm));
+    AddImmediate(Utils::High32Bits(imm));
+    object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
+    return idx;
+  }
+  if (entry.type() == ObjectPoolBuilderEntry::kImmediate128) {
+    ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
+    intptr_t idx = AddImmediate(entry.imm128_.int_storage[0]);
+    AddImmediate(entry.imm128_.int_storage[1]);
+    AddImmediate(entry.imm128_.int_storage[2]);
+    AddImmediate(entry.imm128_.int_storage[3]);
+    object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
+    return idx;
+  }
+#else
+  if (entry.type() == ObjectPoolBuilderEntry::kImmediate128) {
+    ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
+    uword lo64 = static_cast<uword>(entry.imm128_.int_storage[0]) |
+                 (static_cast<uword>(entry.imm128_.int_storage[1]) << 32);
+    uword hi64 = static_cast<uword>(entry.imm128_.int_storage[2]) |
+                 (static_cast<uword>(entry.imm128_.int_storage[3]) << 32);
+    intptr_t idx = AddImmediate(lo64);
+    AddImmediate(hi64);
+    object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
+    return idx;
+  }
+#endif
+
   const intptr_t idx = base_index_ + object_pool_.length();
   object_pool_.Add(entry);
   if (entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable) {
@@ -407,6 +469,22 @@ intptr_t ObjectPoolBuilder::FindImmediate(uword imm) {
                              ObjectPoolBuilderEntry::kNotPatchable));
 }
 
+intptr_t ObjectPoolBuilder::FindImmediate64(uint64_t imm) {
+#if defined(TARGET_ARCH_IS_32_BIT)
+  return FindObject(
+      ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate64,
+                             ObjectPoolBuilderEntry::kNotPatchable));
+#else
+  return FindImmediate(imm);
+#endif
+}
+
+intptr_t ObjectPoolBuilder::FindImmediate128(simd128_value_t imm) {
+  return FindObject(
+      ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate128,
+                             ObjectPoolBuilderEntry::kNotPatchable));
+}
+
 intptr_t ObjectPoolBuilder::FindNativeFunction(
     const ExternalLabel* label,
     ObjectPoolBuilderEntry::Patchability patchable) {

runtime/vm/compiler/assembler/assembler_riscv.cc

@@ -3269,9 +3269,6 @@ void Assembler::LoadPoolPointer(Register pp) {
   set_constant_pool_allowed(pp == PP);
 }
 
-intptr_t Assembler::FindImmediate(int64_t imm) {
-  UNIMPLEMENTED();
-}
 bool Assembler::CanLoadFromObjectPool(const Object& object) const {
   ASSERT(IsOriginalObject(object));
   if (!constant_pool_allowed()) {
@@ -3336,21 +3333,16 @@ void Assembler::LoadDImmediate(FRegister reg, double immd) {
 #endif
   } else {
     ASSERT(constant_pool_allowed());
-#if XLEN >= 64
-    intptr_t index = object_pool_builder().FindImmediate(imm);
+    intptr_t index = object_pool_builder().FindImmediate64(imm);
     intptr_t offset = target::ObjectPool::element_offset(index);
-#else
-    intptr_t lo_index =
-        object_pool_builder().AddImmediate(Utils::Low32Bits(imm));
-    intptr_t hi_index =
-        object_pool_builder().AddImmediate(Utils::High32Bits(imm));
-    ASSERT(lo_index + 1 == hi_index);
-    intptr_t offset = target::ObjectPool::element_offset(lo_index);
-#endif
     LoadDFromOffset(reg, PP, offset);
   }
 }
 
+void Assembler::LoadQImmediate(FRegister reg, simd128_value_t immq) {
+  UNREACHABLE();  // F registers cannot represent SIMD128.
+}
+
 // Load word from pool from the given offset using encoding that
 // InstructionPattern::DecodeLoadWordFromPool can decode.
 //

runtime/vm/compiler/assembler/assembler_riscv.h

@@ -1160,7 +1160,6 @@ class Assembler : public MicroAssembler {
   bool constant_pool_allowed() const { return constant_pool_allowed_; }
   void set_constant_pool_allowed(bool b) { constant_pool_allowed_ = b; }
 
-  intptr_t FindImmediate(int64_t imm);
   bool CanLoadFromObjectPool(const Object& object) const;
   void LoadNativeEntry(Register dst,
                        const ExternalLabel* label,
@@ -1180,6 +1179,7 @@ class Assembler : public MicroAssembler {
   void LoadImmediate(Register reg, intx_t imm);
 
   void LoadDImmediate(FRegister reg, double immd);
+  void LoadQImmediate(FRegister reg, simd128_value_t immq);
 
   // Load word from pool from the given offset using encoding that
   // InstructionPattern::DecodeLoadWordFromPool can decode.

runtime/vm/compiler/assembler/assembler_x64.cc

@@ -1381,17 +1381,13 @@ void Assembler::CompareObject(Register reg, const Object& object) {
   }
 }
 
-intptr_t Assembler::FindImmediate(int64_t imm) {
-  return object_pool_builder().FindImmediate(imm);
-}
-
 void Assembler::LoadImmediate(Register reg, const Immediate& imm) {
   if (imm.value() == 0) {
     xorl(reg, reg);
   } else if (imm.is_int32() || !constant_pool_allowed()) {
     movq(reg, imm);
   } else {
-    const intptr_t idx = FindImmediate(imm.value());
+    const intptr_t idx = object_pool_builder().FindImmediate(imm.value());
     LoadWordFromPoolIndex(reg, idx);
   }
 }
@@ -1410,12 +1406,18 @@ void Assembler::LoadDImmediate(FpuRegister dst, double immediate) {
   if (bits == 0) {
     xorps(dst, dst);
   } else {
-    intptr_t index = FindImmediate(bits);
+    intptr_t index = object_pool_builder().FindImmediate64(bits);
     LoadUnboxedDouble(
         dst, PP, target::ObjectPool::element_offset(index) - kHeapObjectTag);
   }
 }
 
+void Assembler::LoadQImmediate(FpuRegister dst, simd128_value_t immediate) {
+  intptr_t index = object_pool_builder().FindImmediate128(immediate);
+  movups(dst, Address(PP, target::ObjectPool::element_offset(index) -
+                              kHeapObjectTag));
+}
+
 void Assembler::LoadCompressed(Register dest, const Address& slot) {
 #if !defined(DART_COMPRESSED_POINTERS)
   movq(dest, slot);

runtime/vm/compiler/assembler/assembler_x64.h

@@ -781,6 +781,7 @@ class Assembler : public AssemblerBase {
     LoadImmediate(reg, Immediate(immediate));
   }
   void LoadDImmediate(FpuRegister dst, double immediate);
+  void LoadQImmediate(FpuRegister dst, simd128_value_t immediate);
 
   void LoadIsolate(Register dst);
   void LoadIsolateGroup(Register dst);
@@ -1316,7 +1317,6 @@ class Assembler : public AssemblerBase {
  private:
   bool constant_pool_allowed_;
 
-  intptr_t FindImmediate(int64_t imm);
   bool CanLoadFromObjectPool(const Object& object) const;
   void LoadObjectHelper(Register dst, const Object& obj, bool is_unique);
   void LoadWordFromPoolIndex(Register dst, intptr_t index);

runtime/vm/compiler/assembler/object_pool_builder.h

@@ -37,6 +37,13 @@ struct ObjectPoolBuilderEntry {
     // values which become known only at run time.
     kSwitchableCallMissEntryPoint,
     kMegamorphicCallEntryPoint,
+
+  // Used only during object pool building to find duplicates. Become multiple
+  // kImmediate in the final pool.
+#if defined(TARGET_ARCH_IS_32_BIT)
+    kImmediate64,
+#endif
+    kImmediate128,
   };
 
   using TypeBits = BitField<uint8_t, EntryType, 0, 7>;
@@ -46,7 +53,7 @@ struct ObjectPoolBuilderEntry {
     return TypeBits::encode(type) | PatchableBit::encode(patchable);
   }
 
-  ObjectPoolBuilderEntry() : raw_value_(), entry_bits_(0), equivalence_() {}
+  ObjectPoolBuilderEntry() : imm128_(), entry_bits_(0), equivalence_() {}
   ObjectPoolBuilderEntry(const Object* obj, Patchability patchable)
       : ObjectPoolBuilderEntry(obj, obj, patchable) {}
   ObjectPoolBuilderEntry(const Object* obj,
@@ -56,7 +63,19 @@ struct ObjectPoolBuilderEntry {
         entry_bits_(EncodeTraits(kTaggedObject, patchable)),
         equivalence_(eqv) {}
   ObjectPoolBuilderEntry(uword value, EntryType info, Patchability patchable)
-      : raw_value_(value),
+      : imm_(value),
+        entry_bits_(EncodeTraits(info, patchable)),
+        equivalence_() {}
+#if defined(ARCH_IS_32_BIT)
+  ObjectPoolBuilderEntry(uint64_t value, EntryType info, Patchability patchable)
+      : imm64_(value),
+        entry_bits_(EncodeTraits(info, patchable)),
+        equivalence_() {}
+#endif
+  ObjectPoolBuilderEntry(simd128_value_t value,
+                         EntryType info,
+                         Patchability patchable)
+      : imm128_(value),
         entry_bits_(EncodeTraits(info, patchable)),
         equivalence_() {}
 
@@ -66,7 +85,9 @@ struct ObjectPoolBuilderEntry {
 
   union {
     const Object* obj_;
-    uword raw_value_;
+    uword imm_;
+    uint64_t imm64_;
+    simd128_value_t imm128_;
   };
   uint8_t entry_bits_;
   const Object* equivalence_;
@@ -93,8 +114,14 @@ class ObjIndexPair {
     if (key.type() == ObjectPoolBuilderEntry::kTaggedObject) {
       key_.obj_ = key.obj_;
       key_.equivalence_ = key.equivalence_;
+    } else if (key.type() == ObjectPoolBuilderEntry::kImmediate128) {
+      key_.imm128_ = key.imm128_;
+#if defined(TARGET_ARCH_IS_32_BIT)
+    } else if (key.type() == ObjectPoolBuilderEntry::kImmediate64) {
+      key_.imm64_ = key.imm64_;
+#endif
     } else {
-      key_.raw_value_ = key.raw_value_;
+      key_.imm_ = key.imm_;
     }
   }
 
@@ -110,7 +137,18 @@ class ObjIndexPair {
       return IsSameObject(*kv.key_.obj_, *key.obj_) &&
              IsSameObject(*kv.key_.equivalence_, *key.equivalence_);
     }
-    return kv.key_.raw_value_ == key.raw_value_;
+    if (kv.key_.type() == ObjectPoolBuilderEntry::kImmediate128) {
+      return (kv.key_.imm128_.int_storage[0] == key.imm128_.int_storage[0]) &&
+             (kv.key_.imm128_.int_storage[1] == key.imm128_.int_storage[1]) &&
+             (kv.key_.imm128_.int_storage[2] == key.imm128_.int_storage[2]) &&
+             (kv.key_.imm128_.int_storage[3] == key.imm128_.int_storage[3]);
+    }
+#if defined(TARGET_ARCH_IS_32_BIT)
+    if (kv.key_.type() == ObjectPoolBuilderEntry::kImmediate64) {
+      return kv.key_.imm64_ == key.imm64_;
+    }
+#endif
+    return kv.key_.imm_ == key.imm_;
   }
 
  private:
@@ -157,12 +195,16 @@ class ObjectPoolBuilder : public ValueObject {
                      ObjectPoolBuilderEntry::Patchability patchable =
                          ObjectPoolBuilderEntry::kNotPatchable);
   intptr_t AddImmediate(uword imm);
+  intptr_t AddImmediate64(uint64_t imm);
+  intptr_t AddImmediate128(simd128_value_t imm);
 
   intptr_t FindObject(const Object& obj,
                       ObjectPoolBuilderEntry::Patchability patchable =
                           ObjectPoolBuilderEntry::kNotPatchable);
   intptr_t FindObject(const Object& obj, const Object& equivalence);
   intptr_t FindImmediate(uword imm);
+  intptr_t FindImmediate64(uint64_t imm);
+  intptr_t FindImmediate128(simd128_value_t imm);
   intptr_t FindNativeFunction(const ExternalLabel* label,
                               ObjectPoolBuilderEntry::Patchability patchable);
 

runtime/vm/object.cc

@@ -14871,7 +14871,11 @@ ObjectPoolPtr ObjectPool::NewFromBuilder(
     if (type == EntryType::kTaggedObject) {
       result.SetObjectAt(i, *entry.obj_);
     } else {
-      result.SetRawValueAt(i, entry.raw_value_);
+#if defined(TARGET_ARCH_IS_32_BIT)
+      ASSERT(type != EntryType::kImmediate64);
+#endif
+      ASSERT(type != EntryType::kImmediate128);
+      result.SetRawValueAt(i, entry.imm_);
     }
   }
   return result.ptr();