Reapply "[vm, arm64] Add assembler support for tbz and use it for BranchIfSmi."

Skip addition of far branch support to the test harness.

Change-Id: Idec3a644e0dc4165bfaa8daa948f1aee172c1eea
Reviewed-on: https://dart-review.googlesource.com/22740
Reviewed-by: Zach Anderson <zra@google.com>

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -65,6 +65,142 @@ const char* Assembler::FpuRegisterName(FpuRegister reg) {
   return fpu_reg_names[reg];
 }
 
+int32_t Assembler::BindImm19Branch(int64_t position, int64_t dest) {
+  if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
+    // Far branches are enabled, and we can't encode the branch offset in
+    // 19 bits.
+
+    // Grab the guarding branch instruction.
+    const int32_t guard_branch =
+        buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
+
+    // Grab the far branch instruction.
+    const int32_t far_branch =
+        buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
+    const Condition c = DecodeImm19BranchCondition(guard_branch);
+
+    // Grab the link to the next branch.
+    const int32_t next = DecodeImm26BranchOffset(far_branch);
+
+    // dest is the offset is from the guarding branch instruction.
+    // Correct it to be from the following instruction.
+    const int64_t offset = dest - Instr::kInstrSize;
+
+    // Encode the branch.
+    const int32_t encoded_branch = EncodeImm26BranchOffset(offset, far_branch);
+
+    // If the guard branch is conditioned on NV, replace it with a nop.
+    if (c == NV) {
+      buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
+                             Instr::kNopInstruction);
+    }
+
+    // Write the far branch into the buffer and link to the next branch.
+    buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize, encoded_branch);
+    return next;
+  } else if (use_far_branches() && CanEncodeImm19BranchOffset(dest)) {
+    // We assembled a far branch, but we don't need it. Replace it with a near
+    // branch.
+
+    // Grab the guarding branch instruction.
+    const int32_t guard_branch =
+        buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
+
+    // Grab the far branch instruction.
+    const int32_t far_branch =
+        buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
+
+    // Grab the link to the next branch.
+    const int32_t next = DecodeImm26BranchOffset(far_branch);
+
+    // Re-target the guarding branch and flip the conditional sense.
+    int32_t encoded_guard_branch = EncodeImm19BranchOffset(dest, guard_branch);
+    const Condition c = DecodeImm19BranchCondition(encoded_guard_branch);
+    encoded_guard_branch =
+        EncodeImm19BranchCondition(InvertCondition(c), encoded_guard_branch);
+
+    // Write back the re-encoded instructions. The far branch becomes a nop.
+    buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
+                           encoded_guard_branch);
+    buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize,
+                           Instr::kNopInstruction);
+    return next;
+  } else {
+    const int32_t next = buffer_.Load<int32_t>(position);
+    const int32_t encoded = EncodeImm19BranchOffset(dest, next);
+    buffer_.Store<int32_t>(position, encoded);
+    return DecodeImm19BranchOffset(next);
+  }
+}
+
+int32_t Assembler::BindImm14Branch(int64_t position, int64_t dest) {
+  if (use_far_branches() && !CanEncodeImm14BranchOffset(dest)) {
+    // Far branches are enabled, and we can't encode the branch offset in
+    // 14 bits.
+
+    // Grab the guarding branch instruction.
+    const int32_t guard_branch =
+        buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
+
+    // Grab the far branch instruction.
+    const int32_t far_branch =
+        buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
+    const Condition c = DecodeImm14BranchCondition(guard_branch);
+
+    // Grab the link to the next branch.
+    const int32_t next = DecodeImm26BranchOffset(far_branch);
+
+    // dest is the offset is from the guarding branch instruction.
+    // Correct it to be from the following instruction.
+    const int64_t offset = dest - Instr::kInstrSize;
+
+    // Encode the branch.
+    const int32_t encoded_branch = EncodeImm26BranchOffset(offset, far_branch);
+
+    // If the guard branch is conditioned on NV, replace it with a nop.
+    if (c == NV) {
+      buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
+                             Instr::kNopInstruction);
+    }
+
+    // Write the far branch into the buffer and link to the next branch.
+    buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize, encoded_branch);
+    return next;
+  } else if (use_far_branches() && CanEncodeImm14BranchOffset(dest)) {
+    // We assembled a far branch, but we don't need it. Replace it with a near
+    // branch.
+
+    // Grab the guarding branch instruction.
+    const int32_t guard_branch =
+        buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
+
+    // Grab the far branch instruction.
+    const int32_t far_branch =
+        buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
+
+    // Grab the link to the next branch.
+    const int32_t next = DecodeImm26BranchOffset(far_branch);
+
+    // Re-target the guarding branch and flip the conditional sense.
+    int32_t encoded_guard_branch = EncodeImm14BranchOffset(dest, guard_branch);
+    const Condition c = DecodeImm14BranchCondition(encoded_guard_branch);
+    encoded_guard_branch =
+        EncodeImm14BranchCondition(InvertCondition(c), encoded_guard_branch);
+
+    // Write back the re-encoded instructions. The far branch becomes a nop.
+    buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
+                           encoded_guard_branch);
+    buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize,
+                           Instr::kNopInstruction);
+    return next;
+  } else {
+    const int32_t next = buffer_.Load<int32_t>(position);
+    const int32_t encoded = EncodeImm14BranchOffset(dest, next);
+    buffer_.Store<int32_t>(position, encoded);
+    return DecodeImm14BranchOffset(next);
+  }
+}
+
 void Assembler::Bind(Label* label) {
   ASSERT(!label->IsBound());
   const intptr_t bound_pc = buffer_.Size();
@@ -72,73 +208,10 @@ void Assembler::Bind(Label* label) {
   while (label->IsLinked()) {
     const int64_t position = label->Position();
     const int64_t dest = bound_pc - position;
-    if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
-      // Far branches are enabled, and we can't encode the branch offset in
-      // 19 bits.
-
-      // Grab the guarding branch instruction.
-      const int32_t guard_branch =
-          buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
-
-      // Grab the far branch instruction.
-      const int32_t far_branch =
-          buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
-
-      const Condition c = DecodeImm19BranchCondition(guard_branch);
-
-      // Grab the link to the next branch.
-      const int32_t next = DecodeImm26BranchOffset(far_branch);
-
-      // dest is the offset is from the guarding branch instruction.
-      // Correct it to be from the following instruction.
-      const int64_t offset = dest - Instr::kInstrSize;
-
-      // Encode the branch.
-      const int32_t encoded_branch =
-          EncodeImm26BranchOffset(offset, far_branch);
-
-      // If the guard branch is conditioned on NV, replace it with a nop.
-      if (c == NV) {
-        buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
-                               Instr::kNopInstruction);
-      }
-
-      // Write the far branch into the buffer and link to the next branch.
-      buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize, encoded_branch);
-      label->position_ = next;
-    } else if (use_far_branches() && CanEncodeImm19BranchOffset(dest)) {
-      // We assembled a far branch, but we don't need it. Replace it with a near
-      // branch.
-
-      // Grab the guarding branch instruction.
-      const int32_t guard_branch =
-          buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
-
-      // Grab the far branch instruction.
-      const int32_t far_branch =
-          buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
-
-      // Grab the link to the next branch.
-      const int32_t next = DecodeImm26BranchOffset(far_branch);
-
-      // Re-target the guarding branch and flip the conditional sense.
-      int32_t encoded_guard_branch =
-          EncodeImm19BranchOffset(dest, guard_branch);
-      const Condition c = DecodeImm19BranchCondition(encoded_guard_branch);
-      encoded_guard_branch =
-          EncodeImm19BranchCondition(InvertCondition(c), encoded_guard_branch);
-
-      // Write back the re-encoded instructions. The far branch becomes a nop.
-      buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
-                             encoded_guard_branch);
-      buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize,
-                             Instr::kNopInstruction);
-      label->position_ = next;
+    if (IsTestAndBranch(buffer_.Load<int32_t>(position))) {
+      label->position_ = BindImm14Branch(position, dest);
     } else {
-      const int32_t next = buffer_.Load<int32_t>(position);
-      const int32_t encoded = EncodeImm19BranchOffset(dest, next);
-      buffer_.Store<int32_t>(position, encoded);
-      label->position_ = DecodeImm19BranchOffset(next);
+      label->position_ = BindImm19Branch(position, dest);
     }
   }
   label->BindTo(bound_pc);

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -975,6 +975,14 @@ class Assembler : public ValueObject {
     EmitCompareAndBranch(CBNZ, rt, label, sz);
   }
 
+  // Test bit and branch if zero.
+  void tbz(Label* label, Register rt, intptr_t bit_number) {
+    EmitTestAndBranch(TBZ, rt, bit_number, label);
+  }
+  void tbnz(Label* label, Register rt, intptr_t bit_number) {
+    EmitTestAndBranch(TBNZ, rt, bit_number, label);
+  }
+
   // Branch, link, return.
   void br(Register rn) { EmitUnconditionalBranchRegOp(BR, rn); }
   void blr(Register rn) { EmitUnconditionalBranchRegOp(BLR, rn); }
@@ -1314,15 +1322,9 @@ class Assembler : public ValueObject {
     LslImmediate(dst, src, kSmiTagSize);
   }
 
-  void BranchIfNotSmi(Register reg, Label* label) {
-    tsti(reg, Immediate(kSmiTagMask));
-    b(label, NE);
-  }
+  void BranchIfNotSmi(Register reg, Label* label) { tbnz(label, reg, kSmiTag); }
 
-  void BranchIfSmi(Register reg, Label* label) {
-    tsti(reg, Immediate(kSmiTagMask));
-    b(label, EQ);
-  }
+  void BranchIfSmi(Register reg, Label* label) { tbz(label, reg, kSmiTag); }
 
   void Branch(const StubEntry& stub_entry,
               Register pp,
@@ -1683,6 +1685,9 @@ class Assembler : public ValueObject {
     Emit(encoding);
   }
 
+  int32_t BindImm19Branch(int64_t position, int64_t dest);
+  int32_t BindImm14Branch(int64_t position, int64_t dest);
+
   int32_t EncodeImm19BranchOffset(int64_t imm, int32_t instr) {
     if (!CanEncodeImm19BranchOffset(imm)) {
       ASSERT(!use_far_branches());
@@ -1699,6 +1704,22 @@ class Assembler : public ValueObject {
     return static_cast<int64_t>(off);
   }
 
+  int32_t EncodeImm14BranchOffset(int64_t imm, int32_t instr) {
+    if (!CanEncodeImm14BranchOffset(imm)) {
+      ASSERT(!use_far_branches());
+      Thread::Current()->long_jump_base()->Jump(1,
+                                                Object::branch_offset_error());
+    }
+    const int32_t imm32 = static_cast<int32_t>(imm);
+    const int32_t off = (((imm32 >> 2) << kImm14Shift) & kImm14Mask);
+    return (instr & ~kImm14Mask) | off;
+  }
+
+  int64_t DecodeImm14BranchOffset(int32_t instr) {
+    const int32_t off = (((instr & kImm14Mask) >> kImm14Shift) << 18) >> 16;
+    return static_cast<int64_t>(off);
+  }
+
   bool IsConditionalBranch(int32_t instr) {
     return (instr & ConditionalBranchMask) ==
            (ConditionalBranchFixed & ConditionalBranchMask);
@@ -1709,6 +1730,11 @@ class Assembler : public ValueObject {
            (CompareAndBranchFixed & CompareAndBranchMask);
   }
 
+  bool IsTestAndBranch(int32_t instr) {
+    return (instr & TestAndBranchMask) ==
+           (TestAndBranchFixed & TestAndBranchMask);
+  }
+
   Condition DecodeImm19BranchCondition(int32_t instr) {
     if (IsConditionalBranch(instr)) {
       return static_cast<Condition>((instr & kCondMask) >> kCondShift);
@@ -1726,6 +1752,16 @@ class Assembler : public ValueObject {
     return (instr & ~B24) | (cond == EQ ? B24 : 0);  // cbz : cbnz
   }
 
+  Condition DecodeImm14BranchCondition(int32_t instr) {
+    ASSERT(IsTestAndBranch(instr));
+    return (instr & B24) ? EQ : NE;  // tbz : tbnz
+  }
+
+  int32_t EncodeImm14BranchCondition(Condition cond, int32_t instr) {
+    ASSERT(IsTestAndBranch(instr));
+    return (instr & ~B24) | (cond == EQ ? B24 : 0);  // tbz : tbnz
+  }
+
   int32_t EncodeImm26BranchOffset(int64_t imm, int32_t instr) {
     const int32_t imm32 = static_cast<int32_t>(imm);
     const int32_t off = (((imm32 >> 2) << kImm26Shift) & kImm26Mask);
@@ -1750,6 +1786,21 @@ class Assembler : public ValueObject {
     Emit(encoding);
   }
 
+  void EmitTestAndBranchOp(TestAndBranchOp op,
+                           Register rt,
+                           intptr_t bit_number,
+                           int64_t imm) {
+    ASSERT((bit_number >= 0) && (bit_number <= 63));
+    ASSERT(Utils::IsInt(16, imm) && ((imm & 0x3) == 0));
+    ASSERT((rt != CSP) && (rt != R31));
+    const Register crt = ConcreteRegister(rt);
+    const int32_t encoded_offset = EncodeImm14BranchOffset(imm, 0);
+    const int32_t encoding = op | (static_cast<int32_t>(bit_number) << 19) |
+                             (static_cast<int32_t>(crt) << kRtShift) |
+                             encoded_offset;
+    Emit(encoding);
+  }
+
   void EmitConditionalBranchOp(ConditionalBranchOp op,
                                Condition cond,
                                int64_t imm) {
@@ -1764,6 +1815,11 @@ class Assembler : public ValueObject {
     return Utils::IsInt(21, offset);
   }
 
+  bool CanEncodeImm14BranchOffset(int64_t offset) {
+    ASSERT(Utils::IsAligned(offset, 4));
+    return Utils::IsInt(16, offset);
+  }
+
   void EmitConditionalBranch(ConditionalBranchOp op,
                              Condition cond,
                              Label* label) {
@@ -1824,6 +1880,32 @@ class Assembler : public ValueObject {
     }
   }
 
+  void EmitTestAndBranch(TestAndBranchOp op,
+                         Register rt,
+                         intptr_t bit_number,
+                         Label* label) {
+    if (label->IsBound()) {
+      const int64_t dest = label->Position() - buffer_.Size();
+      if (use_far_branches() && !CanEncodeImm14BranchOffset(dest)) {
+        EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
+                            2 * Instr::kInstrSize);
+        b(dest);
+      } else {
+        EmitTestAndBranchOp(op, rt, bit_number, dest);
+      }
+    } else {
+      const int64_t position = buffer_.Size();
+      if (use_far_branches()) {
+        EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
+                            2 * Instr::kInstrSize);
+        b(label->position_);
+      } else {
+        EmitTestAndBranchOp(op, rt, bit_number, label->position_);
+      }
+      label->LinkTo(position);
+    }
+  }
+
   bool CanEncodeImm26BranchOffset(int64_t offset) {
     ASSERT(Utils::IsAligned(offset, 4));
     return Utils::IsInt(26, offset);

runtime/vm/compiler/assembler/assembler_arm64_test.cc

@@ -996,6 +996,74 @@ ASSEMBLER_TEST_RUN(CmpBranchIfNotZeroNotTaken, test) {
   EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry()));
 }
 
+ASSEMBLER_TEST_GENERATE(TstBranchIfZero, assembler) {
+  Label l;
+
+  __ movz(R0, Immediate(42), 0);
+  __ movz(R1, Immediate((0 << 5) | 1), 0);
+
+  __ tbz(&l, R1, 5);
+  __ movz(R0, Immediate(0), 0);
+  __ Bind(&l);
+  __ ret();
+}
+
+ASSEMBLER_TEST_RUN(TstBranchIfZero, test) {
+  typedef int64_t (*Int64Return)() DART_UNUSED;
+  EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry()));
+}
+
+ASSEMBLER_TEST_GENERATE(TstBranchIfZeroNotTaken, assembler) {
+  Label l;
+
+  __ movz(R0, Immediate(0), 0);
+  __ movz(R1, Immediate((1 << 5) | 1), 0);
+
+  __ tbz(&l, R1, 5);
+  __ movz(R0, Immediate(42), 0);
+  __ Bind(&l);
+  __ ret();
+}
+
+ASSEMBLER_TEST_RUN(TstBranchIfZeroNotTaken, test) {
+  typedef int64_t (*Int64Return)() DART_UNUSED;
+  EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry()));
+}
+
+ASSEMBLER_TEST_GENERATE(TstBranchIfNotZero, assembler) {
+  Label l;
+
+  __ movz(R0, Immediate(42), 0);
+  __ movz(R1, Immediate((1 << 5) | 1), 0);
+
+  __ tbnz(&l, R1, 5);
+  __ movz(R0, Immediate(0), 0);
+  __ Bind(&l);
+  __ ret();
+}
+
+ASSEMBLER_TEST_RUN(TstBranchIfNotZero, test) {
+  typedef int64_t (*Int64Return)() DART_UNUSED;
+  EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry()));
+}
+
+ASSEMBLER_TEST_GENERATE(TstBranchIfNotZeroNotTaken, assembler) {
+  Label l;
+
+  __ movz(R0, Immediate(0), 0);
+  __ movz(R1, Immediate((0 << 5) | 1), 0);
+
+  __ tbnz(&l, R1, 5);
+  __ movz(R0, Immediate(42), 0);
+  __ Bind(&l);
+  __ ret();
+}
+
+ASSEMBLER_TEST_RUN(TstBranchIfNotZeroNotTaken, test) {
+  typedef int64_t (*Int64Return)() DART_UNUSED;
+  EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry()));
+}
+
 ASSEMBLER_TEST_GENERATE(FcmpEqBranch, assembler) {
   Label l;
 

runtime/vm/constants_arm64.h

@@ -784,6 +784,7 @@ enum InstructionFields {
   kImm12ShiftBits = 2,
   kImm14Shift = 5,
   kImm14Bits = 14,
+  kImm14Mask = 0x3fff << kImm14Shift,
   kImm16Shift = 5,
   kImm16Bits = 16,
   kImm16Mask = 0xffff << kImm16Shift,