[llvm] [APInt][NFC] Add branch hints to APInt single-word (PR #219437)
Steffen Larsen via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 30 22:37:48 PDT 2026
https://github.com/steffenlarsen updated https://github.com/llvm/llvm-project/pull/219437
>From eeab9be613790199fe999b160138e684d56677cd Mon Sep 17 00:00:00 2001
From: Steffen Holst Larsen <sholstla at amd.com>
Date: Mon, 24 Aug 2026 00:36:11 -0500
Subject: [PATCH] [APInt][NFC] Add branch hints to APInt single-word
The fast-path for APInt is generally done with for
single word integers, i.e. ones that can be
represented in a single 64-bit word. This commit
adds branch hints to the single-word fast-path to
help the compiler generate better code for the
fast-path.
Signed-off-by: Steffen Holst Larsen <sholstla at amd.com>
---
llvm/include/llvm/ADT/APInt.h | 88 +++++++++++++++++------------------
1 file changed, 44 insertions(+), 44 deletions(-)
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 454025b024ae6..8d2806f552f5b 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -129,7 +129,7 @@ class [[nodiscard]] APInt {
}
}
}
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL = val;
if (implicitTrunc || isSigned)
clearUnusedBits();
@@ -171,7 +171,7 @@ class [[nodiscard]] APInt {
/// Copy Constructor.
APInt(const APInt &that) : BitWidth(that.BitWidth) {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL = that.U.VAL;
else
initSlowCase(that);
@@ -368,14 +368,14 @@ class [[nodiscard]] APInt {
bool isAllOnes() const {
if (BitWidth == 0)
return true;
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL == WORDTYPE_MAX >> (APINT_BITS_PER_WORD - BitWidth);
return countTrailingOnesSlowCase() == BitWidth;
}
/// Determine if this value is zero, i.e. all bits are clear.
bool isZero() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL == 0;
return countLeadingZerosSlowCase() == BitWidth;
}
@@ -384,7 +384,7 @@ class [[nodiscard]] APInt {
///
/// This checks to see if the value of this APInt is one.
bool isOne() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL == 1;
return countLeadingZerosSlowCase() == BitWidth - 1;
}
@@ -400,7 +400,7 @@ class [[nodiscard]] APInt {
/// This checks to see if the value of this APInt is the maximum signed
/// value for the APInt's bit width.
bool isMaxSignedValue() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
assert(BitWidth && "zero width values not allowed");
return U.VAL == ((WordType(1) << (BitWidth - 1)) - 1);
}
@@ -418,7 +418,7 @@ class [[nodiscard]] APInt {
/// This checks to see if the value of this APInt is the minimum signed
/// value for the APInt's bit width.
bool isMinSignedValue() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
assert(BitWidth && "zero width values not allowed");
return U.VAL == (WordType(1) << (BitWidth - 1));
}
@@ -435,7 +435,7 @@ class [[nodiscard]] APInt {
///
/// \returns true if the argument APInt value is a power of two > 0.
bool isPowerOf2() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
assert(BitWidth && "zero width values not allowed");
return isPowerOf2_64(U.VAL);
}
@@ -485,7 +485,7 @@ class [[nodiscard]] APInt {
bool isMask(unsigned numBits) const {
assert(numBits != 0 && "numBits must be non-zero");
assert(numBits <= BitWidth && "numBits out of range");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL == (WORDTYPE_MAX >> (APINT_BITS_PER_WORD - numBits));
unsigned Ones = countTrailingOnesSlowCase();
return (numBits == Ones) &&
@@ -496,7 +496,7 @@ class [[nodiscard]] APInt {
/// the least significant bit with the remainder zero.
/// Ex. isMask(0x0000FFFFU) == true.
bool isMask() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return isMask_64(U.VAL);
unsigned Ones = countTrailingOnesSlowCase();
return (Ones > 0) && ((Ones + countLeadingZerosSlowCase()) == BitWidth);
@@ -505,7 +505,7 @@ class [[nodiscard]] APInt {
/// Return true if this APInt value contains a non-empty sequence of ones with
/// the remainder zero.
bool isShiftedMask() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return isShiftedMask_64(U.VAL);
unsigned Ones = countPopulationSlowCase();
unsigned LeadZ = countLeadingZerosSlowCase();
@@ -517,7 +517,7 @@ class [[nodiscard]] APInt {
/// lowest set bit and \p MaskLen is updated to specify the length of the
/// mask, else neither are updated.
bool isShiftedMask(unsigned &MaskIdx, unsigned &MaskLen) const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return isShiftedMask_64(U.VAL, MaskIdx, MaskLen);
unsigned Ones = countPopulationSlowCase();
unsigned LeadZ = countLeadingZerosSlowCase();
@@ -570,7 +570,7 @@ class [[nodiscard]] APInt {
/// This is useful for writing out the APInt in binary form without any
/// conversions.
const uint64_t *getRawData() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return &U.VAL;
return &U.pVal[0];
}
@@ -621,7 +621,7 @@ class [[nodiscard]] APInt {
APInt &operator=(const APInt &RHS) {
// The common case (both source or dest being inline) doesn't require
// allocation or deallocation.
- if (isSingleWord() && RHS.isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord() && RHS.isSingleWord())) {
U.VAL = RHS.U.VAL;
BitWidth = RHS.BitWidth;
return *this;
@@ -639,7 +639,7 @@ class [[nodiscard]] APInt {
return *this;
#endif
assert(this != &that && "Self-move not supported");
- if (!isSingleWord())
+ if (LLVM_UNLIKELY(!isSingleWord()))
delete[] U.pVal;
// Use memcpy so that type based alias analysis sees both VAL and pVal
@@ -659,7 +659,7 @@ class [[nodiscard]] APInt {
///
/// \returns *this after assignment of RHS value.
APInt &operator=(uint64_t RHS) {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL = RHS;
return clearUnusedBits();
}
@@ -676,7 +676,7 @@ class [[nodiscard]] APInt {
/// \returns *this after ANDing with RHS.
APInt &operator&=(const APInt &RHS) {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL &= RHS.U.VAL;
else
andAssignSlowCase(RHS);
@@ -689,7 +689,7 @@ class [[nodiscard]] APInt {
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator&=(uint64_t RHS) {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL &= RHS;
return *this;
}
@@ -706,7 +706,7 @@ class [[nodiscard]] APInt {
/// \returns *this after ORing with RHS.
APInt &operator|=(const APInt &RHS) {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL |= RHS.U.VAL;
else
orAssignSlowCase(RHS);
@@ -719,7 +719,7 @@ class [[nodiscard]] APInt {
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator|=(uint64_t RHS) {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL |= RHS;
return clearUnusedBits();
}
@@ -735,7 +735,7 @@ class [[nodiscard]] APInt {
/// \returns *this after XORing with RHS.
APInt &operator^=(const APInt &RHS) {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL ^= RHS.U.VAL;
else
xorAssignSlowCase(RHS);
@@ -748,7 +748,7 @@ class [[nodiscard]] APInt {
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator^=(uint64_t RHS) {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL ^= RHS;
return clearUnusedBits();
}
@@ -787,7 +787,7 @@ class [[nodiscard]] APInt {
/// \returns *this after shifting left by ShiftAmt
APInt &operator<<=(unsigned ShiftAmt) {
assert(ShiftAmt <= BitWidth && "Invalid shift amount");
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
if (ShiftAmt == BitWidth)
U.VAL = 0;
else
@@ -836,7 +836,7 @@ class [[nodiscard]] APInt {
/// Arithmetic right-shift this APInt by ShiftAmt in place.
void ashrInPlace(unsigned ShiftAmt) {
assert(ShiftAmt <= BitWidth && "Invalid shift amount");
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
int64_t SExtVAL = SignExtend64(U.VAL, BitWidth);
if (ShiftAmt == BitWidth)
U.VAL = SExtVAL >> (APINT_BITS_PER_WORD - 1); // Fill with sign bit.
@@ -860,7 +860,7 @@ class [[nodiscard]] APInt {
/// Logical right-shift this APInt by ShiftAmt in place.
void lshrInPlace(unsigned ShiftAmt) {
assert(ShiftAmt <= BitWidth && "Invalid shift amount");
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
if (ShiftAmt == BitWidth)
U.VAL = 0;
else
@@ -1060,7 +1060,7 @@ class [[nodiscard]] APInt {
/// relationship.
bool operator==(const APInt &RHS) const {
assert(BitWidth == RHS.BitWidth && "Comparison requires equal bit widths");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL == RHS.U.VAL;
return equalSlowCase(RHS);
}
@@ -1253,7 +1253,7 @@ class [[nodiscard]] APInt {
/// between this APInt and RHS that are both set.
bool intersects(const APInt &RHS) const {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return (U.VAL & RHS.U.VAL) != 0;
return intersectsSlowCase(RHS);
}
@@ -1261,7 +1261,7 @@ class [[nodiscard]] APInt {
/// This operation checks that all bits set in this APInt are also set in RHS.
bool isSubsetOf(const APInt &RHS) const {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return (U.VAL & ~RHS.U.VAL) == 0;
return isSubsetOfSlowCase(RHS);
}
@@ -1269,7 +1269,7 @@ class [[nodiscard]] APInt {
/// This operation checks if all bits are set in either this or RHS.
bool isInverseOf(const APInt &RHS) const {
assert(BitWidth == RHS.BitWidth && "Bit widths must be the same");
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return (U.VAL ^ RHS.U.VAL) == llvm::maskTrailingOnes<WordType>(BitWidth);
return isInverseOfSlowCase(RHS);
}
@@ -1338,7 +1338,7 @@ class [[nodiscard]] APInt {
/// Set every bit to 1.
void setAllBits() {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL = WORDTYPE_MAX;
else
// Set all the bits in all the words.
@@ -1351,7 +1351,7 @@ class [[nodiscard]] APInt {
void setBit(unsigned BitPosition) {
assert(BitPosition < BitWidth && "BitPosition out of range");
WordType Mask = maskBit(BitPosition);
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL |= Mask;
else
U.pVal[whichWord(BitPosition)] |= Mask;
@@ -1393,7 +1393,7 @@ class [[nodiscard]] APInt {
if (hiBit <= APINT_BITS_PER_WORD) {
uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - (hiBit - loBit));
mask <<= loBit;
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL |= mask;
else
U.pVal[0] |= mask;
@@ -1415,7 +1415,7 @@ class [[nodiscard]] APInt {
/// Set every bit to 0.
void clearAllBits() {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL = 0;
else
memset(U.pVal, 0, getNumWords() * APINT_WORD_SIZE);
@@ -1427,7 +1427,7 @@ class [[nodiscard]] APInt {
void clearBit(unsigned BitPosition) {
assert(BitPosition < BitWidth && "BitPosition out of range");
WordType Mask = ~maskBit(BitPosition);
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL &= Mask;
else
U.pVal[whichWord(BitPosition)] &= Mask;
@@ -1443,7 +1443,7 @@ class [[nodiscard]] APInt {
if (HiBit <= APINT_BITS_PER_WORD) {
uint64_t Mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - (HiBit - LoBit));
Mask = ~(Mask << LoBit);
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL &= Mask;
else
U.pVal[0] &= Mask;
@@ -1471,7 +1471,7 @@ class [[nodiscard]] APInt {
/// Toggle every bit to its opposite value.
void flipAllBits() {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
U.VAL ^= WORDTYPE_MAX;
clearUnusedBits();
} else {
@@ -1559,7 +1559,7 @@ class [[nodiscard]] APInt {
/// uint64_t. The bitwidth must be <= 64 or the value must fit within a
/// uint64_t. Otherwise an assertion will result.
uint64_t getZExtValue() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return U.VAL;
assert(getActiveBits() <= 64 && "Too many bits for uint64_t");
return U.pVal[0];
@@ -1581,7 +1581,7 @@ class [[nodiscard]] APInt {
/// int64_t. The bit width must be <= 64 or the value must fit within an
/// int64_t. Otherwise an assertion will result.
int64_t getSExtValue() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return SignExtend64(U.VAL, BitWidth);
assert(getSignificantBits() <= 64 && "Too many bits for int64_t");
return int64_t(U.pVal[0]);
@@ -1617,7 +1617,7 @@ class [[nodiscard]] APInt {
/// \returns BitWidth if the value is zero, otherwise returns the number of
/// zeros from the most significant bit to the first one bits.
unsigned countl_zero() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
unsigned unusedBits = APINT_BITS_PER_WORD - BitWidth;
return llvm::countl_zero(U.VAL) - unusedBits;
}
@@ -1634,7 +1634,7 @@ class [[nodiscard]] APInt {
/// \returns 0 if the high order bit is not set, otherwise returns the number
/// of 1 bits from the most significant to the least
unsigned countl_one() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
if (LLVM_UNLIKELY(BitWidth == 0))
return 0;
return llvm::countl_one(U.VAL << (APINT_BITS_PER_WORD - BitWidth));
@@ -1658,7 +1658,7 @@ class [[nodiscard]] APInt {
/// \returns BitWidth if the value is zero, otherwise returns the number of
/// zeros from the least significant bit to the first one bit.
unsigned countr_zero() const {
- if (isSingleWord()) {
+ if (LLVM_LIKELY(isSingleWord())) {
unsigned TrailingZeros = llvm::countr_zero(U.VAL);
return (TrailingZeros > BitWidth ? BitWidth : TrailingZeros);
}
@@ -1675,7 +1675,7 @@ class [[nodiscard]] APInt {
/// \returns BitWidth if the value is all ones, otherwise returns the number
/// of ones from the least significant bit to the first zero bit.
unsigned countr_one() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return llvm::countr_one(U.VAL);
return countTrailingOnesSlowCase();
}
@@ -1689,7 +1689,7 @@ class [[nodiscard]] APInt {
///
/// \returns 0 if the value is zero, otherwise returns the number of set bits.
unsigned popcount() const {
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
return llvm::popcount(U.VAL);
return countPopulationSlowCase();
}
@@ -2010,7 +2010,7 @@ class [[nodiscard]] APInt {
if (LLVM_UNLIKELY(BitWidth == 0))
mask = 0;
- if (isSingleWord())
+ if (LLVM_LIKELY(isSingleWord()))
U.VAL &= mask;
else
U.pVal[getNumWords() - 1] &= mask;
More information about the llvm-commits
mailing list