[llvm] [AggressiveInstCombine] Factor comon code out of tryToRecognizePopCount and tryToRecognizePopCount2n3. (PR #199440)
via llvm-commits
llvm-commits at lists.llvm.org
Sun May 24 12:20:41 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Craig Topper (topperc)
<details>
<summary>Changes</summary>
Both of these patterns end with code to compute the popcount of each byte.
This isn't NFC because tryToRecognizePopCount2n3 handled an alternate pattern for one of the steps and tryToRecognizePopCount used MaskedValueIsZero. The shared code applies these differences to both cases. New tests have been added.
Assisted-by: Claude Sonnet 4.6
---
Full diff: https://github.com/llvm/llvm-project/pull/199440.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp (+67-80)
- (modified) llvm/test/Transforms/AggressiveInstCombine/popcount.ll (+57)
``````````diff
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index f046387def5a4..da03cc045d8c6 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -505,6 +505,60 @@ static void replaceWithPopCount(Instruction &I, Value *Root) {
++NumPopCountRecognized;
}
+// Matches the common innermost steps of the Hacker's Delight popcount idiom:
+// V = ((x + (x >> 4)) & 0x0F...)
+// x = (y & 0x33...) + ((y >> 2) & 0x33...) [or y - 3*((y>>2)&0x33...)]
+// y = Root - ((Root >> 1) & 0x55...)
+// This computes the popcount for each byte.
+// Returns Root on success, nullptr on failure.
+static Value *matchPopCountBytes(Value *V, unsigned Len, const DataLayout &DL) {
+ APInt Mask55 = APInt::getSplat(Len, APInt(8, 0x55));
+ APInt Mask33 = APInt::getSplat(Len, APInt(8, 0x33));
+ APInt Mask0F = APInt::getSplat(Len, APInt(8, 0x0F));
+
+ Value *Add2;
+ // Matching "((x + (x >> 4)) & 0x0F...)".
+ if (!match(V, m_And(m_c_Add(m_LShr(m_Value(Add2), m_SpecificInt(4)),
+ m_Deferred(Add2)),
+ m_SpecificInt(Mask0F))))
+ return nullptr;
+
+ Value *Sub1;
+ llvm::APInt NegThree(Len, -3, /*isSigned=*/true);
+ // Match
+ // x = (x & 0x33333333) + ((x >> 2) & 0x33333333)"
+ // Or
+ // x = x - 3*((x >> 2) & 0x33333333)
+ if (!match(Add2, m_c_Add(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(Mask33)),
+ m_And(m_Deferred(Sub1), m_SpecificInt(Mask33)))) &&
+ !match(Add2, m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(Mask33)),
+ m_SpecificInt(NegThree)),
+ m_Deferred(Sub1))))
+ return nullptr;
+
+ Value *Root, *LShr;
+ const APInt *AndMask;
+ // Matching "x - ((x >> 1) & 0x55...)".
+ if (!match(Sub1,
+ m_Sub(m_Value(Root), m_And(m_Value(LShr, m_LShr(m_Deferred(Root),
+ m_SpecificInt(1))),
+ m_APInt(AndMask)))))
+ return nullptr;
+
+ if (*AndMask != Mask55) {
+ // Accept a narrowed mask if missing bits are known zero in Root>>1.
+ if (!AndMask->isSubsetOf(Mask55))
+ return nullptr;
+ APInt NeededMask = Mask55 & ~*AndMask;
+ if (!MaskedValueIsZero(LShr, NeededMask, SimplifyQuery(DL)))
+ return nullptr;
+ }
+
+ return Root;
+}
+
// Try to recognize below function as popcount intrinsic.
// This is the "best" algorithm from
// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
@@ -526,63 +580,25 @@ static bool tryToRecognizePopCount(Instruction &I) {
unsigned Len = Ty->getScalarSizeInBits();
// FIXME: fix Len == 8 and other irregular type lengths.
- if (!(Len <= 128 && Len > 8 && Len % 8 == 0))
+ if (Len > 128 || Len <= 8 || Len % 8 != 0)
return false;
- APInt Mask55 = APInt::getSplat(Len, APInt(8, 0x55));
- APInt Mask33 = APInt::getSplat(Len, APInt(8, 0x33));
- APInt Mask0F = APInt::getSplat(Len, APInt(8, 0x0F));
APInt Mask01 = APInt::getSplat(Len, APInt(8, 0x01));
- APInt MaskShift = APInt(Len, Len - 8);
Value *Op0 = I.getOperand(0);
Value *Op1 = I.getOperand(1);
Value *MulOp0;
// Matching "(i * 0x01010101...) >> 24".
- if ((match(Op0, m_Mul(m_Value(MulOp0), m_SpecificInt(Mask01)))) &&
- match(Op1, m_SpecificInt(MaskShift))) {
- Value *ShiftOp0;
- // Matching "((i + (i >> 4)) & 0x0F0F0F0F...)".
- if (match(MulOp0, m_And(m_c_Add(m_LShr(m_Value(ShiftOp0), m_SpecificInt(4)),
- m_Deferred(ShiftOp0)),
- m_SpecificInt(Mask0F)))) {
- Value *AndOp0;
- // Matching "(i & 0x33333333...) + ((i >> 2) & 0x33333333...)".
- if (match(ShiftOp0,
- m_c_Add(m_And(m_Value(AndOp0), m_SpecificInt(Mask33)),
- m_And(m_LShr(m_Deferred(AndOp0), m_SpecificInt(2)),
- m_SpecificInt(Mask33))))) {
- Value *Root, *SubOp1;
- // Matching "i - ((i >> 1) & 0x55555555...)".
- const APInt *AndMask;
- if (match(AndOp0, m_Sub(m_Value(Root), m_Value(SubOp1))) &&
- match(SubOp1, m_And(m_LShr(m_Specific(Root), m_SpecificInt(1)),
- m_APInt(AndMask)))) {
- auto CheckAndMask = [&]() {
- if (*AndMask == Mask55)
- return true;
-
- // Exact match failed, see if any bits are known to be 0 where we
- // expect a 1 in the mask.
- if (!AndMask->isSubsetOf(Mask55))
- return false;
-
- APInt NeededMask = Mask55 & ~*AndMask;
- return MaskedValueIsZero(cast<Instruction>(SubOp1)->getOperand(0),
- NeededMask,
- SimplifyQuery(I.getDataLayout()));
- };
-
- if (CheckAndMask()) {
- replaceWithPopCount(I, Root);
- return true;
- }
- }
- }
- }
- }
+ if (!match(Op0, m_Mul(m_Value(MulOp0), m_SpecificInt(Mask01))) ||
+ !match(Op1, m_SpecificInt(Len - 8)))
+ return false;
- return false;
+ Value *Root = matchPopCountBytes(MulOp0, Len, I.getDataLayout());
+ if (!Root)
+ return false;
+
+ replaceWithPopCount(I, Root);
+ return true;
}
// Try to recognize below function as popcount intrinsic.
@@ -727,7 +743,6 @@ static bool tryToRecognizePopCount1(Instruction &I) {
// x = x + (x >> 16);
// return x & 0x0000003F;
// }
-
static bool tryToRecognizePopCount2n3(Instruction &I) {
if (I.getOpcode() != Instruction::And)
return false;
@@ -759,10 +774,6 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
if (!isPowerOf2_32(Len))
return false;
- APInt Mask55 = APInt::getSplat(Len, APInt(8, 0x55));
- APInt Mask33 = APInt::getSplat(Len, APInt(8, 0x33));
- APInt Mask0F = APInt::getSplat(Len, APInt(8, 0x0F));
-
// Number of bits needed to represent Len.
unsigned NumLenBits = Log2_32(Len) + 1;
// The "mask" here really only needs to fulfill two conditions:
@@ -774,8 +785,8 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
if (MaskRes->countTrailingOnes() < NumLenBits || MaskRes->getActiveBits() > 8)
return false;
- Value *Add2;
for (unsigned I = Len; I >= 16; I = I / 2) {
+ Value *Add2;
// Matching "x = x + (x >> I/2)" for I-bit.
if (!match(Add1, m_c_Add(m_LShr(m_Value(Add2), m_SpecificInt(I / 2)),
m_Deferred(Add2))))
@@ -783,32 +794,8 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
Add1 = Add2;
}
- Value *And1 = Add1;
- // Matching "x = (x + (x >> 4)) & 0x0F0F0F0F".
- if (!match(And1, m_And(m_c_Add(m_LShr(m_Value(Add2), m_SpecificInt(4)),
- m_Deferred(Add2)),
- m_SpecificInt(Mask0F))))
- return false;
-
- Value *Sub1;
- llvm::APInt NegThree(/*BitWidth=*/Len, /*Value=*/-3,
- /*isSigned=*/true);
- // x = (x & 0x33333333) + ((x >> 2) & 0x33333333)".
- if (!match(Add2, m_c_Add(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
- m_SpecificInt(Mask33)),
- m_And(m_Deferred(Sub1), m_SpecificInt(Mask33)))) &&
- // Matching "x = x - 3*((x >> 2) & 0x33333333)".
- !match(Add2, m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
- m_SpecificInt(Mask33)),
- m_SpecificInt(NegThree)),
- m_Deferred(Sub1))))
- return false;
-
- Value *Root;
- // x = x - ((x >> 1) & 0x55555555);
- if (!match(Sub1, m_Sub(m_Value(Root),
- m_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
- m_SpecificInt(Mask55)))))
+ Value *Root = matchPopCountBytes(Add1, Len, I.getDataLayout());
+ if (!Root)
return false;
replaceWithPopCount(I, Root);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
index 4b30075b61cad..2d65894906998 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
@@ -90,6 +90,34 @@ define signext i32 @popcount64(i64 %0) {
ret i32 %14
}
+;int popcount64(unsigned long long i) {
+; i = i - ((i >> 1) & 0x5555555555555555);
+; i = i - 3*((i >> 2) & 0x3333333333333333);
+; i = ((i + (i >> 4)) & 0x0F0F0F0F0F0F0F0F);
+; return (i * 0x0101010101010101) >> 56;
+;}
+define signext i32 @popcount64_alt(i64 %0) {
+; CHECK-LABEL: @popcount64_alt(
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0:%.*]])
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %2 = lshr i64 %0, 1
+ %3 = and i64 %2, 6148914691236517205
+ %4 = sub i64 %0, %3
+ %5 = lshr i64 %4, 2
+ %6 = and i64 %5, 3689348814741910323
+ %7 = mul i64 %6, -3
+ %8 = add i64 %7, %4
+ %9 = lshr i64 %8, 4
+ %10 = add i64 %9, %8
+ %11 = and i64 %10, 1085102592571150095
+ %12 = mul i64 %11, 72340172838076673
+ %13 = lshr i64 %12, 56
+ %14 = trunc i64 %13 to i32
+ ret i32 %14
+}
+
;int popcount128(__uint128_t i) {
; __uint128_t x = 0x5555555555555555;
; x <<= 64;
@@ -1495,6 +1523,35 @@ define i64 @popcnt2_64(i64 noundef %0) {
ret i64 %18
}
+; Test that we match the pattern when the input is zext cause some bits in the
+; first mask to be cleared.
+define i64 @popcnt2_64_zext(i32 noundef %0) {
+; CHECK-LABEL: @popcnt2_64_zext(
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[TMP0:%.*]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[ZEXT]])
+; CHECK-NEXT: ret i64 [[TMP2]]
+;
+ %zext = zext i32 %0 to i64
+ %2 = lshr i64 %zext, 1
+ %3 = and i64 %2, 1431655765
+ %4 = sub nsw i64 %zext, %3
+ %5 = and i64 %4, 3689348814741910323
+ %6 = lshr i64 %4, 2
+ %7 = and i64 %6, 3689348814741910323
+ %8 = add nuw nsw i64 %7, %5
+ %9 = lshr i64 %8, 4
+ %10 = add nuw nsw i64 %9, %8
+ %11 = and i64 %10, 1085102592571150095
+ %12 = lshr i64 %11, 8
+ %13 = add nuw nsw i64 %12, %11
+ %14 = lshr i64 %13, 16
+ %15 = add nuw nsw i64 %14, %13
+ %16 = lshr i64 %15, 32
+ %17 = add nuw nsw i64 %16, %15
+ %18 = and i64 %17, 127
+ ret i64 %18
+}
+
define i64 @popcnt2_64_large_mask(i64 noundef %0) {
; CHECK-LABEL: @popcnt2_64_large_mask(
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0:%.*]])
``````````
</details>
https://github.com/llvm/llvm-project/pull/199440
More information about the llvm-commits
mailing list