From c38e19cbbe1f35114eecdc215f5167eda282e57d Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Mon, 7 Oct 2019 18:13:06 +0000 Subject: [PATCH] [vm/compiler] bit utilities Rationale: Useful in the future when compiler wants to constant fold operations related to leading/trailing/counting (non)zeros. https://github.com/dart-lang/sdk/issues/38346 Change-Id: I158aca125d3a15019af20ca7c2a516b74e4d84c2 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119920 Reviewed-by: Alexander Markov Commit-Queue: Aart Bik --- runtime/platform/utils.cc | 78 +++++++++++++++++++ runtime/platform/utils.h | 50 ++++++------ runtime/platform/utils_android.h | 20 ----- runtime/platform/utils_fuchsia.h | 20 ----- runtime/platform/utils_linux.h | 20 ----- runtime/platform/utils_macos.h | 20 ----- runtime/platform/utils_win.h | 24 ------ runtime/vm/bit_set.h | 11 ++- .../vm/compiler/assembler/assembler_arm.cc | 2 +- .../vm/compiler/assembler/assembler_arm64.cc | 4 +- runtime/vm/utils_test.cc | 69 ++++++++++++---- 11 files changed, 165 insertions(+), 153 deletions(-) diff --git a/runtime/platform/utils.cc b/runtime/platform/utils.cc index 3f5091cf186..d6a0f969509 100644 --- a/runtime/platform/utils.cc +++ b/runtime/platform/utils.cc @@ -21,6 +21,32 @@ uintptr_t Utils::RoundUpToPowerOfTwo(uintptr_t x) { return x + 1; } +int Utils::CountOneBits64(uint64_t x) { + // Apparently there are x64 chips without popcount. +#if __GNUC__ && !defined(HOST_ARCH_IA32) && !defined(HOST_ARCH_X64) + return __builtin_popcountll(x); +#else + return CountOneBits32(static_cast(x)) + + CountOneBits32(static_cast(x >> 32)); +#endif +} + +int Utils::CountOneBits32(uint32_t x) { + // Apparently there are x64 chips without popcount. +#if __GNUC__ && !defined(HOST_ARCH_IA32) && !defined(HOST_ARCH_X64) + return __builtin_popcount(x); +#else + // Implementation is from "Hacker's Delight" by Henry S. Warren, Jr., + // figure 5-2, page 66, where the function is called pop. + x = x - ((x >> 1) & 0x55555555); + x = (x & 0x33333333) + ((x >> 2) & 0x33333333); + x = (x + (x >> 4)) & 0x0F0F0F0F; + x = x + (x >> 8); + x = x + (x >> 16); + return static_cast(x & 0x0000003F); +#endif +} + // TODO(koda): Compare to flsll call/intrinsic. int Utils::HighestBit(int64_t v) { uint64_t x = static_cast((v > 0) ? v : -v); @@ -50,6 +76,58 @@ int Utils::HighestBit(int64_t v) { return r; } +int Utils::CountLeadingZeros64(uint64_t x) { +#if defined(ARCH_IS_32_BIT) + const uint32_t x_hi = static_cast(x >> 32); + if (x_hi != 0) { + return CountLeadingZeros32(x_hi); + } + return 32 + CountLeadingZeros32(static_cast(x)); +#elif defined(HOST_OS_WINDOWS) + unsigned long position; // NOLINT + return (_BitScanReverse64(&position, x) == 0) + ? 64 + : 63 - static_cast(position); +#else + return x == 0 ? 64 : __builtin_clzll(x); +#endif +} + +int Utils::CountLeadingZeros32(uint32_t x) { +#if defined(HOST_OS_WINDOWS) + unsigned long position; // NOLINT + return (_BitScanReverse(&position, x) == 0) ? 32 + : 31 - static_cast(position); +#else + return x == 0 ? 32 : __builtin_clz(x); +#endif +} + +int Utils::CountTrailingZeros64(uint64_t x) { +#if defined(ARCH_IS_32_BIT) + const uint32_t x_lo = static_cast(x); + if (x_lo != 0) { + return CountTrailingZeros32(x_lo); + } + return 32 + CountTrailingZeros32(static_cast(x >> 32)); +#elif defined(HOST_OS_WINDOWS) + unsigned long position; // NOLINT + return (_BitScanForward64(&position, x) == 0) ? 64 + : static_cast(position); +#else + return x == 0 ? 64 : __builtin_ctzll(x); +#endif +} + +int Utils::CountTrailingZeros32(uint32_t x) { +#if defined(HOST_OS_WINDOWS) + unsigned long position; // NOLINT + return (_BitScanForward(&position, x) == 0) ? 32 : static_cast(position); +#else + return x == 0 ? 32 : __builtin_ctz(x); +#endif +} + uint64_t Utils::ReverseBits64(uint64_t x) { const uint64_t one = static_cast(1); uint64_t result = 0; diff --git a/runtime/platform/utils.h b/runtime/platform/utils.h index 860c00b1ca5..819206fc985 100644 --- a/runtime/platform/utils.h +++ b/runtime/platform/utils.h @@ -105,31 +105,8 @@ class Utils { static uintptr_t RoundUpToPowerOfTwo(uintptr_t x); - static int CountOneBits32(uint32_t x) { - // Apparently there are x64 chips without popcount. -#if __GNUC__ && !defined(HOST_ARCH_IA32) && !defined(HOST_ARCH_X64) - return __builtin_popcount(x); -#else - // Implementation is from "Hacker's Delight" by Henry S. Warren, Jr., - // figure 5-2, page 66, where the function is called pop. - x = x - ((x >> 1) & 0x55555555); - x = (x & 0x33333333) + ((x >> 2) & 0x33333333); - x = (x + (x >> 4)) & 0x0F0F0F0F; - x = x + (x >> 8); - x = x + (x >> 16); - return static_cast(x & 0x0000003F); -#endif - } - - static int CountOneBits64(uint64_t x) { - // Apparently there are x64 chips without popcount. -#if __GNUC__ && !defined(HOST_ARCH_IA32) && !defined(HOST_ARCH_X64) - return __builtin_popcountll(x); -#else - return CountOneBits32(static_cast(x)) + - CountOneBits32(static_cast(x >> 32)); -#endif - } + static int CountOneBits64(uint64_t x); + static int CountOneBits32(uint32_t x); static int CountOneBitsWord(uword x) { #ifdef ARCH_IS_64_BIT @@ -147,8 +124,27 @@ class Utils { return (value == 0) ? 0 : (Utils::HighestBit(value) + 1); } - static int CountLeadingZeros(uword x); - static int CountTrailingZeros(uword x); + static int CountLeadingZeros64(uint64_t x); + static int CountLeadingZeros32(uint32_t x); + + static int CountLeadingZerosWord(uword x) { +#ifdef ARCH_IS_64_BIT + return CountLeadingZeros64(x); +#else + return CountLeadingZeros32(x); +#endif + } + + static int CountTrailingZeros64(uint64_t x); + static int CountTrailingZeros32(uint32_t x); + + static int CountTrailingZerosWord(uword x) { +#ifdef ARCH_IS_64_BIT + return CountTrailingZeros64(x); +#else + return CountTrailingZeros32(x); +#endif + } static uint64_t ReverseBits64(uint64_t x); static uint32_t ReverseBits32(uint32_t x); diff --git a/runtime/platform/utils_android.h b/runtime/platform/utils_android.h index 6d914156e64..a824443830a 100644 --- a/runtime/platform/utils_android.h +++ b/runtime/platform/utils_android.h @@ -13,26 +13,6 @@ namespace dart { -inline int Utils::CountLeadingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_clzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_clzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - -inline int Utils::CountTrailingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_ctzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_ctzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - inline uint16_t Utils::HostToBigEndian16(uint16_t value) { return htobe16(value); } diff --git a/runtime/platform/utils_fuchsia.h b/runtime/platform/utils_fuchsia.h index c82c797b0bb..7cb381276b8 100644 --- a/runtime/platform/utils_fuchsia.h +++ b/runtime/platform/utils_fuchsia.h @@ -9,26 +9,6 @@ namespace dart { -inline int Utils::CountLeadingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_clzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_clzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - -inline int Utils::CountTrailingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_ctzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_ctzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - inline uint16_t Utils::HostToBigEndian16(uint16_t value) { return htobe16(value); } diff --git a/runtime/platform/utils_linux.h b/runtime/platform/utils_linux.h index 4a49a6a2038..082e4ef0f06 100644 --- a/runtime/platform/utils_linux.h +++ b/runtime/platform/utils_linux.h @@ -13,26 +13,6 @@ namespace dart { -inline int Utils::CountLeadingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_clzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_clzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - -inline int Utils::CountTrailingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_ctzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_ctzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - inline uint16_t Utils::HostToBigEndian16(uint16_t value) { return htobe16(value); } diff --git a/runtime/platform/utils_macos.h b/runtime/platform/utils_macos.h index e64711b7e64..2e3729b4870 100644 --- a/runtime/platform/utils_macos.h +++ b/runtime/platform/utils_macos.h @@ -74,26 +74,6 @@ DEFINE_IS_OS_FUNCS(15, TEST_DEPLOYMENT_TARGET) DEFINE_IS_OS_FUNCS(15, IGNORE_DEPLOYMENT_TARGET) #endif -inline int Utils::CountLeadingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_clzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_clzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - -inline int Utils::CountTrailingZeros(uword x) { -#if defined(ARCH_IS_32_BIT) - return __builtin_ctzl(x); -#elif defined(ARCH_IS_64_BIT) - return __builtin_ctzll(x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif -} - inline uint16_t Utils::HostToBigEndian16(uint16_t value) { return OSSwapHostToBigInt16(value); } diff --git a/runtime/platform/utils_win.h b/runtime/platform/utils_win.h index 9df3f6d422c..c2c90236b24 100644 --- a/runtime/platform/utils_win.h +++ b/runtime/platform/utils_win.h @@ -14,30 +14,6 @@ namespace dart { -inline int Utils::CountLeadingZeros(uword x) { - unsigned long position; // NOLINT -#if defined(ARCH_IS_32_BIT) - _BitScanReverse(&position, x); -#elif defined(ARCH_IS_64_BIT) - _BitScanReverse64(&position, x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif - return kBitsPerWord - static_cast(position) - 1; -} - -inline int Utils::CountTrailingZeros(uword x) { - unsigned long result; // NOLINT -#if defined(ARCH_IS_32_BIT) - _BitScanForward(&result, x); -#elif defined(ARCH_IS_64_BIT) - _BitScanForward64(&result, x); -#else -#error Architecture is not 32-bit or 64-bit. -#endif - return static_cast(result); -} - // WARNING: The below functions assume host is always Little Endian! inline uint16_t Utils::HostToBigEndian16(uint16_t value) { diff --git a/runtime/vm/bit_set.h b/runtime/vm/bit_set.h index 4e716c1184d..c16cceabf71 100644 --- a/runtime/vm/bit_set.h +++ b/runtime/vm/bit_set.h @@ -41,12 +41,13 @@ class BitSet { intptr_t w = i >> kBitsPerWordLog2; uword mask = ~static_cast(0) << (i & (kBitsPerWord - 1)); if ((data_[w] & mask) != 0) { - uword tz = Utils::CountTrailingZeros(data_[w] & mask); + uword tz = Utils::CountTrailingZerosWord(data_[w] & mask); return (w << kBitsPerWordLog2) + tz; } while (++w < kLengthInWords) { if (data_[w] != 0) { - return (w << kBitsPerWordLog2) + Utils::CountTrailingZeros(data_[w]); + return (w << kBitsPerWordLog2) + + Utils::CountTrailingZerosWord(data_[w]); } } return -1; @@ -56,7 +57,8 @@ class BitSet { for (int w = kLengthInWords - 1; w >= 0; --w) { uword d = data_[w]; if (d != 0) { - return ((w + 1) << kBitsPerWordLog2) - Utils::CountLeadingZeros(d) - 1; + return ((w + 1) << kBitsPerWordLog2) - Utils::CountLeadingZerosWord(d) - + 1; } } return -1; @@ -79,7 +81,8 @@ class BitSet { return -1; } else { // Bitlength incl. w, minus leading zeroes of w, minus 1 to 0-based index. - return ((w + 1) << kBitsPerWordLog2) - Utils::CountLeadingZeros(bits) - 1; + return ((w + 1) << kBitsPerWordLog2) - + Utils::CountLeadingZerosWord(bits) - 1; } } diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index 8ce69364ef6..e2e07a3ae8e 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -1732,7 +1732,7 @@ Register AllocateRegister(RegList* used) { return (free == 0) ? kNoRegister : UseRegister( - static_cast(Utils::CountTrailingZeros(free)), + static_cast(Utils::CountTrailingZerosWord(free)), used); } diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index d973ac9bc5a..ad41da5153e 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -281,8 +281,8 @@ bool Operand::IsImmLogical(uint64_t value, uint8_t width, Operand* imm_op) { int lead_zero = CountLeadingZeros(value, width); int lead_one = CountLeadingZeros(~value, width); - int trail_zero = Utils::CountTrailingZeros(value); - int trail_one = Utils::CountTrailingZeros(~value); + int trail_zero = Utils::CountTrailingZerosWord(value); + int trail_one = Utils::CountTrailingZerosWord(~value); int set_bits = CountOneBits(value, width); // The fixed bits in the immediate s field. diff --git a/runtime/vm/utils_test.cc b/runtime/vm/utils_test.cc index 03e18efb623..a115a4f7b95 100644 --- a/runtime/vm/utils_test.cc +++ b/runtime/vm/utils_test.cc @@ -116,7 +116,6 @@ VM_UNIT_TEST_CASE(CountOneBits64) { EXPECT_EQ(4, Utils::CountOneBits64(DART_UINT64_C(0x10101010))); EXPECT_EQ(8, Utils::CountOneBits64(DART_UINT64_C(0x03030303))); EXPECT_EQ(32, Utils::CountOneBits64(DART_UINT64_C(0xFFFFFFFF))); - EXPECT_EQ(2, Utils::CountOneBits64(DART_UINT64_C(0x0000001000000010))); EXPECT_EQ(2, Utils::CountOneBits64(DART_UINT64_C(0x0001000000010000))); EXPECT_EQ(2, Utils::CountOneBits64(DART_UINT64_C(0x1000000010000000))); @@ -133,7 +132,6 @@ VM_UNIT_TEST_CASE(CountOneBitsWord) { EXPECT_EQ(4, Utils::CountOneBitsWord(0x10101010)); EXPECT_EQ(8, Utils::CountOneBitsWord(0x03030303)); EXPECT_EQ(32, Utils::CountOneBitsWord(0xFFFFFFFF)); - #if defined(ARCH_IS_64_BIT) EXPECT_EQ(2, Utils::CountOneBitsWord(0x0000001000000010)); EXPECT_EQ(2, Utils::CountOneBitsWord(0x0001000000010000)); @@ -144,20 +142,54 @@ VM_UNIT_TEST_CASE(CountOneBitsWord) { #endif } -VM_UNIT_TEST_CASE(CountZeros) { - EXPECT_EQ(0, Utils::CountTrailingZeros(0x1)); - EXPECT_EQ(kBitsPerWord - 1, Utils::CountLeadingZeros(0x1)); - EXPECT_EQ(1, Utils::CountTrailingZeros(0x2)); - EXPECT_EQ(kBitsPerWord - 2, Utils::CountLeadingZeros(0x2)); - EXPECT_EQ(0, Utils::CountTrailingZeros(0x3)); - EXPECT_EQ(kBitsPerWord - 2, Utils::CountLeadingZeros(0x3)); - EXPECT_EQ(2, Utils::CountTrailingZeros(0x4)); - EXPECT_EQ(kBitsPerWord - 3, Utils::CountLeadingZeros(0x4)); - EXPECT_EQ(0, Utils::CountTrailingZeros(kUwordMax)); - EXPECT_EQ(0, Utils::CountLeadingZeros(kUwordMax)); +VM_UNIT_TEST_CASE(CountTrailingZeros32) { + EXPECT_EQ(0, Utils::CountTrailingZeros32(0x1)); + EXPECT_EQ(1, Utils::CountTrailingZeros32(0x2)); + EXPECT_EQ(4, Utils::CountTrailingZeros32(0x0f0f0)); + EXPECT_EQ(31, Utils::CountTrailingZeros32(0x80000000)); + EXPECT_EQ(32, Utils::CountTrailingZeros32(0x0)); +} + +VM_UNIT_TEST_CASE(CountTrailingZeros64) { + EXPECT_EQ(0, Utils::CountTrailingZeros64(0x1)); + EXPECT_EQ(1, Utils::CountTrailingZeros64(0x2)); + EXPECT_EQ(4, Utils::CountTrailingZeros64(0x0f0f0)); + EXPECT_EQ(63, Utils::CountTrailingZeros64(0x8000000000000000LLU)); + EXPECT_EQ(64, Utils::CountTrailingZeros64(0x0)); +} + +VM_UNIT_TEST_CASE(CountLeadingZeros32) { + EXPECT_EQ(32, Utils::CountLeadingZeros32(0x0)); + EXPECT_EQ(31, Utils::CountLeadingZeros32(0x1)); + EXPECT_EQ(4, Utils::CountLeadingZeros32(0x0F0F0000)); + EXPECT_EQ(1, Utils::CountLeadingZeros32(0x7FFFFFFF)); + EXPECT_EQ(0, Utils::CountLeadingZeros32(0xFFFFFFFF)); +} + +VM_UNIT_TEST_CASE(CountLeadingZeros64) { + EXPECT_EQ(64, Utils::CountLeadingZeros64(0x0)); + EXPECT_EQ(63, Utils::CountLeadingZeros64(0x1)); + EXPECT_EQ(4, Utils::CountLeadingZeros64(0x0F0F000000000000LLU)); + EXPECT_EQ(1, Utils::CountLeadingZeros64(0x7FFFFFFFFFFFFFFFLLU)); + EXPECT_EQ(0, Utils::CountLeadingZeros64(0xFFFFFFFFFFFFFFFFLLU)); +} + +VM_UNIT_TEST_CASE(CountZerosWord) { + EXPECT_EQ(kBitsPerWord, Utils::CountTrailingZerosWord(0x0)); + EXPECT_EQ(kBitsPerWord, Utils::CountLeadingZerosWord(0x0)); + EXPECT_EQ(0, Utils::CountTrailingZerosWord(0x1)); + EXPECT_EQ(kBitsPerWord - 1, Utils::CountLeadingZerosWord(0x1)); + EXPECT_EQ(1, Utils::CountTrailingZerosWord(0x2)); + EXPECT_EQ(kBitsPerWord - 2, Utils::CountLeadingZerosWord(0x2)); + EXPECT_EQ(0, Utils::CountTrailingZerosWord(0x3)); + EXPECT_EQ(kBitsPerWord - 2, Utils::CountLeadingZerosWord(0x3)); + EXPECT_EQ(2, Utils::CountTrailingZerosWord(0x4)); + EXPECT_EQ(kBitsPerWord - 3, Utils::CountLeadingZerosWord(0x4)); + EXPECT_EQ(0, Utils::CountTrailingZerosWord(kUwordMax)); + EXPECT_EQ(0, Utils::CountLeadingZerosWord(kUwordMax)); static const uword kTopBit = static_cast(1) << (kBitsPerWord - 1); - EXPECT_EQ(kBitsPerWord - 1, Utils::CountTrailingZeros(kTopBit)); - EXPECT_EQ(0, Utils::CountLeadingZeros(kTopBit)); + EXPECT_EQ(kBitsPerWord - 1, Utils::CountTrailingZerosWord(kTopBit)); + EXPECT_EQ(0, Utils::CountLeadingZerosWord(kTopBit)); } VM_UNIT_TEST_CASE(ReverseBits32) { @@ -176,6 +208,13 @@ VM_UNIT_TEST_CASE(ReverseBits64) { EXPECT_EQ(0x8f7b3d591e6a2c48LLU, Utils::ReverseBits64(0x123456789abcdef1LLU)); } +VM_UNIT_TEST_CASE(ReverseBitsWord) { + const uword kOne = static_cast(1); + const uword kTopBit = kOne << (kBitsPerWord - 1); + EXPECT_EQ(kTopBit, Utils::ReverseBitsWord(kOne)); + EXPECT_EQ(kOne, Utils::ReverseBitsWord(kTopBit)); +} + VM_UNIT_TEST_CASE(IsInt) { EXPECT(Utils::IsInt(8, 16)); EXPECT(Utils::IsInt(8, 127));