[llvm] [AggressiveInstCombine] POPCNT generation for bit-count pattern (PR #180917)
Rohit Aggarwal via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 12 08:22:52 PDT 2026
https://github.com/rohitaggarwal007 updated https://github.com/llvm/llvm-project/pull/180917
>From b1ae92139795d550cc94d425432c90cab1501241 Mon Sep 17 00:00:00 2001
From: Rohit Aggarwal <Rohit.Aggarwal at amd.com>
Date: Wed, 21 Jan 2026 12:52:41 +0530
Subject: [PATCH 1/3] [AggressiveInstCombine] POPCNT generation for bit-count
pattern
---
.../AggressiveInstCombine.cpp | 90 +++++++++++++++++++
.../AggressiveInstCombine/popcount.ll | 48 ++++++++++
2 files changed, 138 insertions(+)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 3341368208c24..5ca2fa14e1533 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -372,6 +372,95 @@ static bool tryToRecognizePopCount(Instruction &I) {
return false;
}
+// Try to recognize below function as popcount intrinsic.
+// https://doc.lagout.org/security/Hackers%20Delight.pdf
+// Also used in TargetLowering::expandCTPOP().
+//
+// int popcnt(unsigned x) {
+// x = x - ((x >> 1) & 0x55555555);
+// x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
+// x = (x + (x >> 4)) & 0x0F0F0F0F;
+// x = x + (x >> 8);
+// x = x + (x >> 16);
+// return x & 0x0000003F;
+// }
+
+// int popcnt(unsigned x) {
+// x = x - ((x >> 1) & 0x55555555);
+// x = x - 3*((x >> 2) & 0x33333333);
+// x = (x + (x >> 4)) & 0x0F0F0F0F;
+// x = x + (x >> 8);
+// x = x + (x >> 16);
+// return x & 0x0000003F;
+// }
+
+static bool tryToRecognizePopCount2n3(Instruction &I) {
+ if (I.getOpcode() != Instruction::And)
+ return false;
+
+ Type *Ty = I.getType();
+ if (!Ty->isIntOrIntVectorTy())
+ return false;
+
+ unsigned Len = Ty->getScalarSizeInBits();
+ if (!(Len <= 128 && Len > 8 && Len % 8 == 0))
+ return false;
+
+ Value *Op0 = I.getOperand(0);
+ Value *Op1 = I.getOperand(1);
+ Value *LShrOp0;
+ Value *AddOp1;
+ // Matching "x & 0x0000003F".
+ if ((match(Op0, m_Add(m_Value(LShrOp0), m_Value(AddOp1)))) &&
+ match(Op1, m_SpecificInt(63))) {
+ Value *LShr1;
+ Value *And1;
+ // Matching "x = x + (x >> 16)".
+ if (match(LShrOp0, m_LShr(m_Add(m_Value(LShr1), m_Value(And1)),
+ m_SpecificInt(16)))) {
+ Value *Add2;
+ // Matching " x = x + (x >> 8)".
+ if (match(LShr1, m_LShr(m_Deferred(And1), m_SpecificInt(8))) &&
+ match(And1, m_c_And(m_Value(Add2), m_SpecificInt(252645135)))) {
+ Value *Add3;
+ // Matching "x = (x + (x >> 4)) & 0x0F0F0F0F".
+ if (match(Add2, m_c_Add(m_LShr(m_Value(Add3), m_SpecificInt(4)),
+ m_Deferred(Add3)))) {
+ Value *Sub1;
+ llvm::APInt NegThree(/*BitWidth=*/32, /*Value=*/-3,
+ /*isSigned=*/true);
+ // x = (x & 0x33333333) + ((x >> 2) & 0x33333333)".
+ if (match(Add3,
+ m_c_Add(
+ m_c_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(858993459)),
+ m_c_And(m_Deferred(Sub1), m_SpecificInt(858993459)))) ||
+ // Matching "x = x - 3*((x >> 2) & 0x33333333)".
+ match(Add3,
+ m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(858993459)),
+ m_SpecificInt(NegThree)),
+ m_Deferred(Sub1)))) {
+ Value *Root;
+ if (match(Sub1,
+ m_Sub(m_Value(Root),
+ m_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
+ m_SpecificInt(1431655765))))) {
+ LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
+ IRBuilder<> Builder(&I);
+ I.replaceAllUsesWith(Builder.CreateIntrinsic(
+ Intrinsic::ctpop, I.getType(), {Root}));
+ ++NumPopCountRecognized;
+ return true;
+ }
+ }
+ }
+ }
+ }
+ }
+ return false;
+}
+
/// Fold smin(smax(fptosi(x), C1), C2) to llvm.fptosi.sat(x), providing C1 and
/// C2 saturate the value of the fp conversion. The transform is not reversable
/// as the fptosi.sat is more defined than the input - all values produce a
@@ -1826,6 +1915,7 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
MadeChange |= tryToRecognizePopCount(I);
+ MadeChange |= tryToRecognizePopCount2n3(I);
MadeChange |= tryToFPToSat(I, TTI);
MadeChange |= tryToRecognizeTableBasedCttz(I, DL);
MadeChange |= foldConsecutiveLoads(I, DL, TTI, AA, DT);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
index f56cab1503531..2c9fde6608984 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
@@ -239,3 +239,51 @@ define i32 @popcount64_mask(i64 %x) {
%13 = trunc nuw nsw i64 %12 to i32
ret i32 %13
}
+
+define dso_local noundef range(i32 0, 64) i32 @popcnt2(i32 noundef %0) local_unnamed_addr {
+; CHECK-LABEL: define dso_local noundef range(i32 0, 64) i32 @popcnt2(
+; CHECK-SAME: i32 noundef [[TMP0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP0]])
+; CHECK-NEXT: ret i32 [[TMP2]]
+;
+ %2 = lshr i32 %0, 1
+ %3 = and i32 %2, 1431655765
+ %4 = sub i32 %0, %3
+ %5 = and i32 %4, 858993459
+ %6 = lshr i32 %4, 2
+ %7 = and i32 %6, 858993459
+ %8 = add nuw nsw i32 %7, %5
+ %9 = lshr i32 %8, 4
+ %10 = add nuw nsw i32 %9, %8
+ %11 = and i32 %10, 252645135
+ %12 = lshr i32 %11, 8
+ %13 = add nuw nsw i32 %12, %11
+ %14 = lshr i32 %13, 16
+ %15 = add nuw nsw i32 %14, %13
+ %16 = and i32 %15, 63
+ ret i32 %16
+}
+
+define dso_local noundef range(i32 0, 64) i32 @popcnt3(i32 noundef %0) local_unnamed_addr {
+; CHECK-LABEL: define dso_local noundef range(i32 0, 64) i32 @popcnt3(
+; CHECK-SAME: i32 noundef [[TMP0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP0]])
+; CHECK-NEXT: ret i32 [[TMP16]]
+;
+ %2 = lshr i32 %0, 1
+ %3 = and i32 %2, 1431655765
+ %4 = sub i32 %0, %3
+ %5 = lshr i32 %4, 2
+ %6 = and i32 %5, 858993459
+ %7 = mul i32 %6, -3
+ %8 = add i32 %7, %4
+ %9 = lshr i32 %8, 4
+ %10 = add i32 %9, %8
+ %11 = and i32 %10, 252645135
+ %12 = lshr i32 %11, 8
+ %13 = add nuw nsw i32 %12, %11
+ %14 = lshr i32 %13, 16
+ %15 = add nuw nsw i32 %14, %13
+ %16 = and i32 %15, 63
+ ret i32 %16
+}
>From 13d9f15f4ab023f5fbe4e541835c4bf34da0172e Mon Sep 17 00:00:00 2001
From: Rohit Aggarwal <Rohit.Aggarwal at amd.com>
Date: Wed, 25 Feb 2026 17:00:56 +0530
Subject: [PATCH 2/3] Implement review comments
---
.../AggressiveInstCombine.cpp | 127 +++---
.../AggressiveInstCombine/popcount.ll | 384 +++++++++++++++++-
2 files changed, 456 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 5ca2fa14e1533..e1ad6458187ce 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -403,61 +403,90 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
return false;
unsigned Len = Ty->getScalarSizeInBits();
- if (!(Len <= 128 && Len > 8 && Len % 8 == 0))
+ if (!(Len == 16 || Len == 32 || Len == 64))
return false;
+ APInt Mask55 = APInt::getSplat(Len, APInt(8, 0x55));
+ APInt Mask33 = APInt::getSplat(Len, APInt(8, 0x33));
+ APInt Mask0F = APInt::getSplat(Len, APInt(8, 0x0F));
+
+ APInt MaskRes = APInt(Len, 2 * Len - 1);
Value *Op0 = I.getOperand(0);
Value *Op1 = I.getOperand(1);
- Value *LShrOp0;
- Value *AddOp1;
- // Matching "x & 0x0000003F".
- if ((match(Op0, m_Add(m_Value(LShrOp0), m_Value(AddOp1)))) &&
- match(Op1, m_SpecificInt(63))) {
- Value *LShr1;
- Value *And1;
- // Matching "x = x + (x >> 16)".
- if (match(LShrOp0, m_LShr(m_Add(m_Value(LShr1), m_Value(And1)),
- m_SpecificInt(16)))) {
- Value *Add2;
- // Matching " x = x + (x >> 8)".
- if (match(LShr1, m_LShr(m_Deferred(And1), m_SpecificInt(8))) &&
- match(And1, m_c_And(m_Value(Add2), m_SpecificInt(252645135)))) {
- Value *Add3;
- // Matching "x = (x + (x >> 4)) & 0x0F0F0F0F".
- if (match(Add2, m_c_Add(m_LShr(m_Value(Add3), m_SpecificInt(4)),
- m_Deferred(Add3)))) {
- Value *Sub1;
- llvm::APInt NegThree(/*BitWidth=*/32, /*Value=*/-3,
- /*isSigned=*/true);
- // x = (x & 0x33333333) + ((x >> 2) & 0x33333333)".
- if (match(Add3,
- m_c_Add(
- m_c_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
- m_SpecificInt(858993459)),
- m_c_And(m_Deferred(Sub1), m_SpecificInt(858993459)))) ||
- // Matching "x = x - 3*((x >> 2) & 0x33333333)".
- match(Add3,
- m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
- m_SpecificInt(858993459)),
- m_SpecificInt(NegThree)),
- m_Deferred(Sub1)))) {
- Value *Root;
- if (match(Sub1,
- m_Sub(m_Value(Root),
- m_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
- m_SpecificInt(1431655765))))) {
- LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
- IRBuilder<> Builder(&I);
- I.replaceAllUsesWith(Builder.CreateIntrinsic(
- Intrinsic::ctpop, I.getType(), {Root}));
- ++NumPopCountRecognized;
- return true;
- }
- }
- }
- }
+
+ // Matching "x & 0x001F" (16-bit).
+ // Matching "x & 0x0000003F" (32-bit).
+ // Matching "x & 0x0000007F" (64-bit).
+ if (!(match(Op0, m_SpecificInt(MaskRes)) ||
+ match(Op1, m_SpecificInt(MaskRes)))) {
+ return false;
+ }
+
+ // Get the value being masked
+ Value *Add1 = (match(Op0, m_SpecificInt(MaskRes))) ? Op1 : Op0;
+
+ // Matching "x = x + (x >> 32)" for 64-bit.
+ Value *Add2 = Add1;
+ if (Len == 64) {
+ if (!match(Add1, m_c_Add(m_LShr(m_Value(Add2), m_SpecificInt(32)),
+ m_Deferred(Add2)))) {
+ return false;
}
}
+
+ // Matching "x = x + (x >> 16)" for 32-bit and 64-bit.
+ Value *Add3 = Add2;
+ if (Len == 32 || Len == 64) {
+ if (!match(Add2, m_c_Add(m_LShr(m_Value(Add3), m_SpecificInt(16)),
+ m_Deferred(Add3)))) {
+ return false;
+ }
+ }
+
+ // Matching "x = x + (x >> 8)".
+ Value *And1;
+ if (!match(Add3, m_c_Add(m_LShr(m_Value(And1), m_SpecificInt(8)),
+ m_Deferred(And1)))) {
+ return false;
+ }
+
+ // Matching "x = (x + (x >> 4)) & 0x0F0F0F0F".
+ Value *Add4;
+ if (!match(And1, m_c_And(m_c_Add(m_LShr(m_Value(Add4), m_SpecificInt(4)),
+ m_Deferred(Add4)),
+ m_SpecificInt(Mask0F)))) {
+ return false;
+ }
+
+ Value *Sub1;
+ llvm::APInt NegThree(/*BitWidth=*/Len, /*Value=*/-3,
+ /*isSigned=*/true);
+ // x = (x & 0x33333333) + ((x >> 2) & 0x33333333)".
+ if (!(match(Add4,
+ m_c_Add(m_c_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(Mask33)),
+ m_c_And(m_Deferred(Sub1), m_SpecificInt(Mask33)))) ||
+ // Matching "x = x - 3*((x >> 2) & 0x33333333)".
+ match(Add4, m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(Mask33)),
+ m_SpecificInt(NegThree)),
+ m_Deferred(Sub1))))) {
+ return false;
+ }
+
+ Value *Root;
+ // x = x - ((x >> 1) & 0x55555555);
+ if (match(Sub1, m_Sub(m_Value(Root),
+ m_c_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
+ m_SpecificInt(Mask55))))) {
+ LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
+ IRBuilder<> Builder(&I);
+ I.replaceAllUsesWith(
+ Builder.CreateIntrinsic(Intrinsic::ctpop, I.getType(), {Root}));
+ ++NumPopCountRecognized;
+ return true;
+ }
+
return false;
}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
index 2c9fde6608984..9909854a8ac07 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
@@ -240,9 +240,9 @@ define i32 @popcount64_mask(i64 %x) {
ret i32 %13
}
-define dso_local noundef range(i32 0, 64) i32 @popcnt2(i32 noundef %0) local_unnamed_addr {
-; CHECK-LABEL: define dso_local noundef range(i32 0, 64) i32 @popcnt2(
-; CHECK-SAME: i32 noundef [[TMP0:%.*]]) local_unnamed_addr {
+define i32 @popcnt2_32(i32 noundef %0) {
+; CHECK-LABEL: define i32 @popcnt2_32(
+; CHECK-SAME: i32 noundef [[TMP0:%.*]]) {
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP0]])
; CHECK-NEXT: ret i32 [[TMP2]]
;
@@ -264,9 +264,9 @@ define dso_local noundef range(i32 0, 64) i32 @popcnt2(i32 noundef %0) local_unn
ret i32 %16
}
-define dso_local noundef range(i32 0, 64) i32 @popcnt3(i32 noundef %0) local_unnamed_addr {
-; CHECK-LABEL: define dso_local noundef range(i32 0, 64) i32 @popcnt3(
-; CHECK-SAME: i32 noundef [[TMP0:%.*]]) local_unnamed_addr {
+define i32 @popcnt3_32(i32 noundef %0) {
+; CHECK-LABEL: define i32 @popcnt3_32(
+; CHECK-SAME: i32 noundef [[TMP0:%.*]]) {
; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP0]])
; CHECK-NEXT: ret i32 [[TMP16]]
;
@@ -287,3 +287,375 @@ define dso_local noundef range(i32 0, 64) i32 @popcnt3(i32 noundef %0) local_unn
%16 = and i32 %15, 63
ret i32 %16
}
+
+; 16-bit scalar popcount
+define i16 @popcnt2_16(i16 noundef %0) {
+; CHECK-LABEL: @popcnt2_16(
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.ctpop.i16(i16 [[TMP0:%.*]])
+; CHECK-NEXT: ret i16 [[TMP2]]
+;
+ %2 = lshr i16 %0, 1
+ %3 = and i16 %2, 21845
+ %4 = sub i16 %0, %3
+ %5 = and i16 %4, 13107
+ %6 = lshr i16 %4, 2
+ %7 = and i16 %6, 13107
+ %8 = add nuw nsw i16 %7, %5
+ %9 = lshr i16 %8, 4
+ %10 = add nuw nsw i16 %9, %8
+ %11 = and i16 %10, 3855
+ %12 = lshr i16 %11, 8
+ %13 = add nuw nsw i16 %12, %11
+ %14 = and i16 %13, 31
+ ret i16 %14
+}
+
+; 64-bit scalar popcount
+define i64 @popcnt2_64(i64 noundef %0) {
+; CHECK-LABEL: @popcnt2_64(
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0:%.*]])
+; CHECK-NEXT: ret i64 [[TMP2]]
+;
+ %2 = lshr i64 %0, 1
+ %3 = and i64 %2, 6148914691236517205
+ %4 = sub i64 %0, %3
+ %5 = and i64 %4, 3689348814741910323
+ %6 = lshr i64 %4, 2
+ %7 = and i64 %6, 3689348814741910323
+ %8 = add nuw nsw i64 %7, %5
+ %9 = lshr i64 %8, 4
+ %10 = add nuw nsw i64 %9, %8
+ %11 = and i64 %10, 1085102592571150095
+ %12 = lshr i64 %11, 8
+ %13 = add nuw nsw i64 %12, %11
+ %14 = lshr i64 %13, 16
+ %15 = add nuw nsw i64 %14, %13
+ %16 = lshr i64 %15, 32
+ %17 = add nuw nsw i64 %16, %15
+ %18 = and i64 %17, 127
+ ret i64 %18
+}
+
+; 16-bit vector popcount
+define <8 x i16> @popcnt2_16vec(<8 x i16> %0) {
+; CHECK-LABEL: @popcnt2_16vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> [[TMP0:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %2 = lshr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = and <8 x i16> %2, <i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845>
+ %4 = sub <8 x i16> %0, %3
+ %5 = and <8 x i16> %4, <i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107>
+ %6 = lshr <8 x i16> %4, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+ %7 = and <8 x i16> %6, <i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107>
+ %8 = add nuw nsw <8 x i16> %7, %5
+ %9 = lshr <8 x i16> %8, <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4>
+ %10 = add nuw nsw <8 x i16> %9, %8
+ %11 = and <8 x i16> %10, <i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855>
+ %12 = lshr <8 x i16> %11, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+ %13 = add nuw nsw <8 x i16> %12, %11
+ %14 = and <8 x i16> %13, <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>
+ ret <8 x i16> %14
+}
+
+; 32-bit vector popcount (variant 2) - 4 elements
+define <4 x i32> @popcnt2_32vec(<4 x i32> %0) {
+; CHECK-LABEL: @popcnt2_32vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP0:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %2 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+ %3 = and <4 x i32> %2, <i32 1431655765, i32 1431655765, i32 1431655765, i32 1431655765>
+ %4 = sub <4 x i32> %0, %3
+ %5 = and <4 x i32> %4, <i32 858993459, i32 858993459, i32 858993459, i32 858993459>
+ %6 = lshr <4 x i32> %4, <i32 2, i32 2, i32 2, i32 2>
+ %7 = and <4 x i32> %6, <i32 858993459, i32 858993459, i32 858993459, i32 858993459>
+ %8 = add nuw nsw <4 x i32> %7, %5
+ %9 = lshr <4 x i32> %8, <i32 4, i32 4, i32 4, i32 4>
+ %10 = add nuw nsw <4 x i32> %9, %8
+ %11 = and <4 x i32> %10, <i32 252645135, i32 252645135, i32 252645135, i32 252645135>
+ %12 = lshr <4 x i32> %11, <i32 8, i32 8, i32 8, i32 8>
+ %13 = add nuw nsw <4 x i32> %12, %11
+ %14 = lshr <4 x i32> %13, <i32 16, i32 16, i32 16, i32 16>
+ %15 = add nuw nsw <4 x i32> %14, %13
+ %16 = and <4 x i32> %15, <i32 63, i32 63, i32 63, i32 63>
+ ret <4 x i32> %16
+}
+
+; 64-bit vector popcount
+define <2 x i64> @popcnt2_64vec(<2 x i64> %0) {
+; CHECK-LABEL: @popcnt2_64vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[TMP0:%.*]])
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %2 = lshr <2 x i64> %0, <i64 1, i64 1>
+ %3 = and <2 x i64> %2, <i64 6148914691236517205, i64 6148914691236517205>
+ %4 = sub <2 x i64> %0, %3
+ %5 = and <2 x i64> %4, <i64 3689348814741910323, i64 3689348814741910323>
+ %6 = lshr <2 x i64> %4, <i64 2, i64 2>
+ %7 = and <2 x i64> %6, <i64 3689348814741910323, i64 3689348814741910323>
+ %8 = add nuw nsw <2 x i64> %7, %5
+ %9 = lshr <2 x i64> %8, <i64 4, i64 4>
+ %10 = add nuw nsw <2 x i64> %9, %8
+ %11 = and <2 x i64> %10, <i64 1085102592571150095, i64 1085102592571150095>
+ %12 = lshr <2 x i64> %11, <i64 8, i64 8>
+ %13 = add nuw nsw <2 x i64> %12, %11
+ %14 = lshr <2 x i64> %13, <i64 16, i64 16>
+ %15 = add nuw nsw <2 x i64> %14, %13
+ %16 = lshr <2 x i64> %15, <i64 32, i64 32>
+ %17 = add nuw nsw <2 x i64> %16, %15
+ %18 = and <2 x i64> %17, <i64 127, i64 127>
+ ret <2 x i64> %18
+}
+
+; 16-bit scalar popcount (variant 3 - using multiply by -3)
+define i16 @popcnt3_16(i16 noundef %0) {
+; CHECK-LABEL: @popcnt3_16(
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.ctpop.i16(i16 [[TMP0:%.*]])
+; CHECK-NEXT: ret i16 [[TMP2]]
+;
+ %2 = lshr i16 %0, 1
+ %3 = and i16 %2, 21845
+ %4 = sub i16 %0, %3
+ %5 = lshr i16 %4, 2
+ %6 = and i16 %5, 13107
+ %7 = mul i16 %6, -3
+ %8 = add i16 %7, %4
+ %9 = lshr i16 %8, 4
+ %10 = add i16 %9, %8
+ %11 = and i16 %10, 3855
+ %12 = lshr i16 %11, 8
+ %13 = add nuw nsw i16 %12, %11
+ %14 = and i16 %13, 31
+ ret i16 %14
+}
+
+; 64-bit scalar popcount (variant 3 - using multiply by -3)
+define i64 @popcnt3_64(i64 noundef %0) {
+; CHECK-LABEL: @popcnt3_64(
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0:%.*]])
+; CHECK-NEXT: ret i64 [[TMP2]]
+;
+ %2 = lshr i64 %0, 1
+ %3 = and i64 %2, 6148914691236517205
+ %4 = sub i64 %0, %3
+ %5 = lshr i64 %4, 2
+ %6 = and i64 %5, 3689348814741910323
+ %7 = mul i64 %6, -3
+ %8 = add i64 %7, %4
+ %9 = lshr i64 %8, 4
+ %10 = add i64 %9, %8
+ %11 = and i64 %10, 1085102592571150095
+ %12 = lshr i64 %11, 8
+ %13 = add nuw nsw i64 %12, %11
+ %14 = lshr i64 %13, 16
+ %15 = add nuw nsw i64 %14, %13
+ %16 = lshr i64 %15, 32
+ %17 = add nuw nsw i64 %16, %15
+ %18 = and i64 %17, 127
+ ret i64 %18
+}
+
+; 16-bit vector popcount (variant 3 - using multiply by -3) - 8 elements
+define <8 x i16> @popcnt3_16vec(<8 x i16> %0) {
+; CHECK-LABEL: @popcnt3_16vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> [[TMP0:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %2 = lshr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = and <8 x i16> %2, <i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845, i16 21845>
+ %4 = sub <8 x i16> %0, %3
+ %5 = lshr <8 x i16> %4, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+ %6 = and <8 x i16> %5, <i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107, i16 13107>
+ %7 = mul <8 x i16> %6, <i16 -3, i16 -3, i16 -3, i16 -3, i16 -3, i16 -3, i16 -3, i16 -3>
+ %8 = add <8 x i16> %7, %4
+ %9 = lshr <8 x i16> %8, <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4>
+ %10 = add <8 x i16> %9, %8
+ %11 = and <8 x i16> %10, <i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855, i16 3855>
+ %12 = lshr <8 x i16> %11, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+ %13 = add nuw nsw <8 x i16> %12, %11
+ %14 = and <8 x i16> %13, <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>
+ ret <8 x i16> %14
+}
+
+; 32-bit vector popcount (variant 3 - using multiply by -3) - 4 elements
+define <4 x i32> @popcnt3_32vec(<4 x i32> %0) {
+; CHECK-LABEL: @popcnt3_32vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP0:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %2 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+ %3 = and <4 x i32> %2, <i32 1431655765, i32 1431655765, i32 1431655765, i32 1431655765>
+ %4 = sub <4 x i32> %0, %3
+ %5 = lshr <4 x i32> %4, <i32 2, i32 2, i32 2, i32 2>
+ %6 = and <4 x i32> %5, <i32 858993459, i32 858993459, i32 858993459, i32 858993459>
+ %7 = mul <4 x i32> %6, <i32 -3, i32 -3, i32 -3, i32 -3>
+ %8 = add <4 x i32> %7, %4
+ %9 = lshr <4 x i32> %8, <i32 4, i32 4, i32 4, i32 4>
+ %10 = add <4 x i32> %9, %8
+ %11 = and <4 x i32> %10, <i32 252645135, i32 252645135, i32 252645135, i32 252645135>
+ %12 = lshr <4 x i32> %11, <i32 8, i32 8, i32 8, i32 8>
+ %13 = add nuw nsw <4 x i32> %12, %11
+ %14 = lshr <4 x i32> %13, <i32 16, i32 16, i32 16, i32 16>
+ %15 = add nuw nsw <4 x i32> %14, %13
+ %16 = and <4 x i32> %15, <i32 63, i32 63, i32 63, i32 63>
+ ret <4 x i32> %16
+}
+
+; 64-bit vector popcount (variant 3 - using multiply by -3) - 2 elements
+define <2 x i64> @popcnt3_64vec(<2 x i64> %0) {
+; CHECK-LABEL: @popcnt3_64vec(
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[TMP0:%.*]])
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %2 = lshr <2 x i64> %0, <i64 1, i64 1>
+ %3 = and <2 x i64> %2, <i64 6148914691236517205, i64 6148914691236517205>
+ %4 = sub <2 x i64> %0, %3
+ %5 = lshr <2 x i64> %4, <i64 2, i64 2>
+ %6 = and <2 x i64> %5, <i64 3689348814741910323, i64 3689348814741910323>
+ %7 = mul <2 x i64> %6, <i64 -3, i64 -3>
+ %8 = add <2 x i64> %7, %4
+ %9 = lshr <2 x i64> %8, <i64 4, i64 4>
+ %10 = add <2 x i64> %9, %8
+ %11 = and <2 x i64> %10, <i64 1085102592571150095, i64 1085102592571150095>
+ %12 = lshr <2 x i64> %11, <i64 8, i64 8>
+ %13 = add nuw nsw <2 x i64> %12, %11
+ %14 = lshr <2 x i64> %13, <i64 16, i64 16>
+ %15 = add nuw nsw <2 x i64> %14, %13
+ %16 = lshr <2 x i64> %15, <i64 32, i64 32>
+ %17 = add nuw nsw <2 x i64> %16, %15
+ %18 = and <2 x i64> %17, <i64 127, i64 127>
+ ret <2 x i64> %18
+}
+
+; Negative test case for popcnt2 i8 - wrong constant (should NOT optimize)
+define i8 @popcnt2_negative_i8(i8 noundef %0) {
+; CHECK-LABEL: @popcnt2_negative_i8(
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i8 [[TMP0:%.*]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = and i8 [[TMP2]], 85
+; CHECK-NEXT: [[TMP4:%.*]] = sub i8 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = and i8 [[TMP4]], 51
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i8 [[TMP4]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = and i8 [[TMP6]], 51
+; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw i8 [[TMP7]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = lshr i8 [[TMP8]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i8 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = and i8 [[TMP10]], 15
+; CHECK-NEXT: ret i8 [[TMP11]]
+;
+ %2 = lshr i8 %0, 1
+ %3 = and i8 %2, 85
+ %4 = sub i8 %0, %3
+ %5 = and i8 %4, 51
+ %6 = lshr i8 %4, 2
+ %7 = and i8 %6, 51
+ %8 = add nuw nsw i8 %7, %5
+ %9 = lshr i8 %8, 4
+ %10 = add nuw nsw i8 %9, %8
+ %11 = and i8 %10, 15
+ ret i8 %11
+}
+
+; Negative test case for popcnt3 i8 - using wrong multiplier (should NOT optimize)
+define i8 @popcnt3_negative_i8(i8 noundef %0) {
+; CHECK-LABEL: @popcnt3_negative_i8(
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i8 [[TMP0:%.*]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = and i8 [[TMP2]], 85
+; CHECK-NEXT: [[TMP4:%.*]] = sub i8 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i8 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = and i8 [[TMP5]], 51
+; CHECK-NEXT: [[TMP7:%.*]] = mul i8 [[TMP6]], -3
+; CHECK-NEXT: [[TMP8:%.*]] = add i8 [[TMP7]], [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = lshr i8 [[TMP8]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = add i8 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = and i8 [[TMP10]], 15
+; CHECK-NEXT: ret i8 [[TMP11]]
+;
+ %2 = lshr i8 %0, 1
+ %3 = and i8 %2, 85
+ %4 = sub i8 %0, %3
+ %5 = lshr i8 %4, 2
+ %6 = and i8 %5, 51
+ %7 = mul i8 %6, -3
+ %8 = add i8 %7, %4
+ %9 = lshr i8 %8, 4
+ %10 = add i8 %9, %8
+ %11 = and i8 %10, 15
+ ret i8 %11
+}
+
+; Negative test case for popcnt2 i32 - wrong constant in second step (should NOT optimize)
+define i32 @popcnt2_negative_i32(i32 noundef %0) {
+; CHECK-LABEL: @popcnt2_negative_i32(
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP0:%.*]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[TMP2]], 1431655765
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[TMP4]], 858993460
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[TMP6]], 858993459
+; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw i32 [[TMP7]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i32 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = and i32 [[TMP10]], 252645135
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 8
+; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i32 [[TMP13]], 16
+; CHECK-NEXT: [[TMP15:%.*]] = add nuw nsw i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT: [[TMP16:%.*]] = and i32 [[TMP15]], 63
+; CHECK-NEXT: ret i32 [[TMP16]]
+;
+ %2 = lshr i32 %0, 1
+ %3 = and i32 %2, 1431655765
+ %4 = sub i32 %0, %3
+ %5 = and i32 %4, 858993460 ; Wrong constant (should be 858993459)
+ %6 = lshr i32 %4, 2
+ %7 = and i32 %6, 858993459
+ %8 = add nuw nsw i32 %7, %5
+ %9 = lshr i32 %8, 4
+ %10 = add nuw nsw i32 %9, %8
+ %11 = and i32 %10, 252645135
+ %12 = lshr i32 %11, 8
+ %13 = add nuw nsw i32 %12, %11
+ %14 = lshr i32 %13, 16
+ %15 = add nuw nsw i32 %14, %13
+ %16 = and i32 %15, 63
+ ret i32 %16
+}
+
+; Negative test case for popcnt3 i32 - using wrong multiplier (should NOT optimize)
+define i32 @popcnt3_negative_i32(i32 noundef %0) {
+; CHECK-LABEL: @popcnt3_negative_i32(
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP0:%.*]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[TMP2]], 1431655765
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 858993459
+; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], -2
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = and i32 [[TMP10]], 252645135
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 8
+; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i32 [[TMP13]], 16
+; CHECK-NEXT: [[TMP15:%.*]] = add nuw nsw i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT: [[TMP16:%.*]] = and i32 [[TMP15]], 63
+; CHECK-NEXT: ret i32 [[TMP16]]
+;
+ %2 = lshr i32 %0, 1
+ %3 = and i32 %2, 1431655765
+ %4 = sub i32 %0, %3
+ %5 = lshr i32 %4, 2
+ %6 = and i32 %5, 858993459
+ %7 = mul i32 %6, -2 ; Wrong multiplier (should be -3)
+ %8 = add i32 %7, %4
+ %9 = lshr i32 %8, 4
+ %10 = add i32 %9, %8
+ %11 = and i32 %10, 252645135
+ %12 = lshr i32 %11, 8
+ %13 = add nuw nsw i32 %12, %11
+ %14 = lshr i32 %13, 16
+ %15 = add nuw nsw i32 %14, %13
+ %16 = and i32 %15, 63
+ ret i32 %16
+}
>From 74dde1550f505c68572d25f08bc06b5f5f9b787e Mon Sep 17 00:00:00 2001
From: Rohit Aggarwal <Rohit.Aggarwal at amd.com>
Date: Thu, 12 Mar 2026 20:52:03 +0530
Subject: [PATCH 3/3] Fix the review comments
---
.../AggressiveInstCombine.cpp | 37 +++++++------------
1 file changed, 13 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index e1ad6458187ce..8da569c660121 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -373,7 +373,7 @@ static bool tryToRecognizePopCount(Instruction &I) {
}
// Try to recognize below function as popcount intrinsic.
-// https://doc.lagout.org/security/Hackers%20Delight.pdf
+// Ref. Hackers Delight
// Also used in TargetLowering::expandCTPOP().
//
// int popcnt(unsigned x) {
@@ -403,7 +403,7 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
return false;
unsigned Len = Ty->getScalarSizeInBits();
- if (!(Len == 16 || Len == 32 || Len == 64))
+ if (Len != 16 && Len != 32 && Len != 64)
return false;
APInt Mask55 = APInt::getSplat(Len, APInt(8, 0x55));
APInt Mask33 = APInt::getSplat(Len, APInt(8, 0x33));
@@ -411,19 +411,9 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
APInt MaskRes = APInt(Len, 2 * Len - 1);
- Value *Op0 = I.getOperand(0);
- Value *Op1 = I.getOperand(1);
-
- // Matching "x & 0x001F" (16-bit).
- // Matching "x & 0x0000003F" (32-bit).
- // Matching "x & 0x0000007F" (64-bit).
- if (!(match(Op0, m_SpecificInt(MaskRes)) ||
- match(Op1, m_SpecificInt(MaskRes)))) {
+ Value *Add1;
+ if (!match(&I, m_And(m_Value(Add1), m_SpecificInt(MaskRes))))
return false;
- }
-
- // Get the value being masked
- Value *Add1 = (match(Op0, m_SpecificInt(MaskRes))) ? Op1 : Op0;
// Matching "x = x + (x >> 32)" for 64-bit.
Value *Add2 = Add1;
@@ -436,7 +426,7 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
// Matching "x = x + (x >> 16)" for 32-bit and 64-bit.
Value *Add3 = Add2;
- if (Len == 32 || Len == 64) {
+ if (Len >= 32) {
if (!match(Add2, m_c_Add(m_LShr(m_Value(Add3), m_SpecificInt(16)),
m_Deferred(Add3)))) {
return false;
@@ -452,9 +442,9 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
// Matching "x = (x + (x >> 4)) & 0x0F0F0F0F".
Value *Add4;
- if (!match(And1, m_c_And(m_c_Add(m_LShr(m_Value(Add4), m_SpecificInt(4)),
- m_Deferred(Add4)),
- m_SpecificInt(Mask0F)))) {
+ if (!match(And1, m_And(m_c_Add(m_LShr(m_Value(Add4), m_SpecificInt(4)),
+ m_Deferred(Add4)),
+ m_SpecificInt(Mask0F)))) {
return false;
}
@@ -462,10 +452,9 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
llvm::APInt NegThree(/*BitWidth=*/Len, /*Value=*/-3,
/*isSigned=*/true);
// x = (x & 0x33333333) + ((x >> 2) & 0x33333333)".
- if (!(match(Add4,
- m_c_Add(m_c_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
- m_SpecificInt(Mask33)),
- m_c_And(m_Deferred(Sub1), m_SpecificInt(Mask33)))) ||
+ if (!(match(Add4, m_c_Add(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
+ m_SpecificInt(Mask33)),
+ m_And(m_Deferred(Sub1), m_SpecificInt(Mask33)))) ||
// Matching "x = x - 3*((x >> 2) & 0x33333333)".
match(Add4, m_Add(m_Mul(m_And(m_LShr(m_Value(Sub1), m_SpecificInt(2)),
m_SpecificInt(Mask33)),
@@ -477,8 +466,8 @@ static bool tryToRecognizePopCount2n3(Instruction &I) {
Value *Root;
// x = x - ((x >> 1) & 0x55555555);
if (match(Sub1, m_Sub(m_Value(Root),
- m_c_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
- m_SpecificInt(Mask55))))) {
+ m_And(m_LShr(m_Deferred(Root), m_SpecificInt(1)),
+ m_SpecificInt(Mask55))))) {
LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
IRBuilder<> Builder(&I);
I.replaceAllUsesWith(
More information about the llvm-commits
mailing list