[llvm-branch-commits] [llvm] release/23.x: [AggressiveInstCombine] Avoid converting first table element when folding table-based cttz (#211202) (PR #211742)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 24 00:46:20 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: llvmbot
<details>
<summary>Changes</summary>
Backport 9edfdc231c9811124bbb7e3943c578c538d46aba
Requested by: @<!-- -->antoniofrighetto
---
Full diff: https://github.com/llvm/llvm-project/pull/211742.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp (+10-11)
- (modified) llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll (+64-21)
- (modified) llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll (+2-2)
- (modified) llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll (+2-2)
- (modified) llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll (+2-2)
``````````diff
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index cef41c971aa18..dbd9df84071ba 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -987,25 +987,26 @@ static bool tryToRecognizeTableBasedCttz(LoadInst *LI, Type *AccessType,
ConstantInt *ZeroTableElem = cast<ConstantInt>(
ConstantFoldLoadFromConst(GVTable->getInitializer(), AccessType, DL));
- bool DefinedForZero = ZeroTableElem->getZExtValue() == InputBits;
+ bool DefinedForZero = ZeroTableElem->equalsInt(InputBits);
IRBuilder<> B(LI);
ConstantInt *BoolConst = B.getInt1(!DefinedForZero);
Type *XType = X1->getType();
auto Cttz = B.CreateIntrinsic(Intrinsic::cttz, {XType}, {X1, BoolConst});
- Value *ZExtOrTrunc = nullptr;
+ Value *Res = B.CreateZExtOrTrunc(Cttz, AccessType);
- if (DefinedForZero) {
- ZExtOrTrunc = B.CreateZExtOrTrunc(Cttz, AccessType);
- } else {
+ if (!DefinedForZero) {
// If the value in elem 0 isn't the same as InputBits, we still want to
- // produce the value from the table.
+ // produce the value from the table. Emit the select in AccessType with elem
+ // 0 unchanged, as the table's element type may be wider than the input
+ // type (and directly truncating ZeroTableElem into the input type could
+ // incorrectly drop bits).
auto Cmp = B.CreateICmpEQ(X1, ConstantInt::get(XType, 0));
- auto Select = B.CreateSelect(Cmp, B.CreateZExt(ZeroTableElem, XType), Cttz);
+ Res = B.CreateSelect(Cmp, ZeroTableElem, Res);
// The true branch of select handles the cttz(0) case, which is rare.
if (!ProfcheckDisableMetadataFixes) {
- if (Instruction *SelectI = dyn_cast<Instruction>(Select))
+ if (Instruction *SelectI = dyn_cast<Instruction>(Res))
SelectI->setMetadata(
LLVMContext::MD_prof,
MDBuilder(SelectI->getContext()).createUnlikelyBranchWeights());
@@ -1013,11 +1014,9 @@ static bool tryToRecognizeTableBasedCttz(LoadInst *LI, Type *AccessType,
// NOTE: If the table[0] is 0, but the cttz(0) is defined by the Target
// it should be handled as: `cttz(x) & (typeSize - 1)`.
-
- ZExtOrTrunc = B.CreateZExtOrTrunc(Select, AccessType);
}
- LI->replaceAllUsesWith(ZExtOrTrunc);
+ LI->replaceAllUsesWith(Res);
return true;
}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll
index a7d34461283e6..58fb8c0a47746 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll
@@ -96,9 +96,9 @@ define i32 @ctz1(i32 %x) !prof !0 {
; CHECK: !prof [[PROF_0:![0-9]+]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]], !prof [[PROF_1:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -118,9 +118,9 @@ define i32 @ctz1_nusw(i32 %x) {
; CHECK-LABEL: @ctz1_nusw(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -230,9 +230,9 @@ define i32 @ctz4(i64 %b) {
; CHECK-LABEL: @ctz4(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[B:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[B]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -251,9 +251,9 @@ define i32 @ctz5(i32 %x) {
; CHECK-LABEL: @ctz5(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -275,9 +275,9 @@ define i32 @ctz6(i64 %n) {
; CHECK-LABEL: @ctz6(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[N:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 63, i64 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -297,7 +297,7 @@ define i32 @ctz8(i32 %v) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[V:%.*]], i1 true)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[V]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 31, i32 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 31, i32 [[TMP0]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
@@ -317,9 +317,9 @@ define i32 @ctz9(i32 %x) {
; CHECK-LABEL: @ctz9(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -339,9 +339,9 @@ define i32 @ctz1_with_i8_gep(i32 %x) {
; CHECK-LABEL: @ctz1_with_i8_gep(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -409,9 +409,9 @@ define i32 @cttz_i16_via_i128(i16 noundef %x) {
; CHECK-LABEL: @cttz_i16_via_i128(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i16 [[TMP0]] to i8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP3]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV6:%.*]] = zext i8 [[TMP1]] to i32
; CHECK-NEXT: ret i32 [[CONV6]]
;
@@ -460,9 +460,9 @@ define i128 @ctz1_i128(i32 %x) {
; CHECK-LABEL: @ctz1_i128(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP0]] to i128
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]]
-; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i128
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i128 0, i128 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: ret i128 [[TMP3]]
;
entry:
@@ -482,9 +482,9 @@ define i32 @src(i128 noundef %x) {
; CHECK-LABEL: @src(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP3:%.*]] = call i128 @llvm.cttz.i128(i128 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i128 [[TMP3]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i128 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i128 0, i128 [[TMP3]]
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i128 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
@@ -500,6 +500,49 @@ entry:
ret i32 %conv
}
+ at table.wider = constant [16 x i32] [i32 15, i32 0, i32 1, i32 4, i32 2, i32 8, i32 5, i32 10, i32 14, i32 3, i32 7, i32 9, i32 13, i32 6, i32 12, i32 11]
+define i32 @cttz_accesstype_wider_nonconst(i16 %x) {
+; CHECK-LABEL: @cttz_accesstype_wider_nonconst(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i16 [[X]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 15, i32 [[TMP1]], !prof [[PROF_1:![0-9]+]]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %neg = sub i16 0, %x
+ %and = and i16 %neg, %x
+ %mul = mul i16 %and, 2479
+ %shr = lshr i16 %mul, 11
+ %mask = and i16 %shr, 15
+ %idx = zext i16 %mask to i64
+ %gep = getelementptr inbounds [16 x i32], ptr @table.wider, i64 0, i64 %idx
+ %res = load i32, ptr %gep, align 4
+ ret i32 %res
+}
+
+ at table.zeroelement.wider = constant [27 x i128] [i128 18446744073709551632, i128 0, i128 1, i128 0, i128 0, i128 2, i128 7, i128 0, i128 0, i128 5, i128 3, i128 11, i128 13, i128 8, i128 0, i128 0, i128 15, i128 0, i128 0, i128 6, i128 4, i128 10, i128 12, i128 0, i128 14, i128 0, i128 9]
+define i128 @cttz_zeroelement_wider(i16 %x) {
+; CHECK-LABEL: @cttz_zeroelement_wider(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i128
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i16 [[X]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i128 18446744073709551632, i128 [[TMP1]], !prof [[PROF_1:![0-9]+]]
+; CHECK-NEXT: ret i128 [[TMP3]]
+;
+entry:
+ %neg = sub i16 0, %x
+ %and = and i16 %neg, %x
+ %mul = mul i16 %and, 2667
+ %shr = lshr i16 %mul, 11
+ %idx = zext i16 %shr to i64
+ %gep = getelementptr inbounds [27 x i128], ptr @table.zeroelement.wider, i64 0, i64 %idx
+ %res = load i128, ptr %gep, align 16
+ ret i128 %res
+}
+
!0 = !{!"function_entry_count", i64 1000}
; CHECK: [[PROF_0]] = !{!"function_entry_count", i64 1000}
; CHECK: [[PROF_1]] = !{!"branch_weights", i32 1, i32 1048575}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll
index 0e5c4f0fb3288..c943cb422666f 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll
@@ -26,9 +26,9 @@ define i32 @ctz6(ptr nocapture readonly %b) !prof !0 {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[B:%.*]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP0]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 0, i64 [[TMP1]], !prof [[PROF_1:![0-9]+]]
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP2]], i32 0, i32 [[TMP3]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: ret i32 [[TMP4]]
;
entry:
diff --git a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll
index a7732f0a2e095..016c0830076fc 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll
@@ -26,9 +26,9 @@ define i32 @test() !prof !0 {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @x, align 4
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[TMP0]], i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP1]] to i8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]], !prof [[PROF_1:![0-9]+]]
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP2]], i8 0, i8 [[TMP3]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP4]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
diff --git a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll
index 5f9b4cefa8638..50c378182ff31 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll
@@ -8,9 +8,9 @@ define i32 @ctz1(i32 %x) !prof !0 {
; CHECK: !prof [[PROF_0:![0-9]+]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP0]] to i8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP0]], !prof [[PROF_1:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP1]], i8 0, i8 [[TMP2]], !prof [[PROF_1:![0-9]+]]
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: ret i32 [[CONV]]
;
``````````
</details>
https://github.com/llvm/llvm-project/pull/211742
More information about the llvm-branch-commits
mailing list