[clang] c053daa - [ConstraintElim] Add NUW for mul/shl with NSW and non-positive ops. (#219663)
via cfe-commits
cfe-commits at lists.llvm.org
Sun Aug 30 08:11:56 PDT 2026
Author: Florian Hahn
Date: 2026-08-30T16:11:51+01:00
New Revision: c053daa736f6ad30c638725985f64d28280f47ae
URL: https://github.com/llvm/llvm-project/commit/c053daa736f6ad30c638725985f64d28280f47ae
DIFF: https://github.com/llvm/llvm-project/commit/c053daa736f6ad30c638725985f64d28280f47ae.diff
LOG: [ConstraintElim] Add NUW for mul/shl with NSW and non-positive ops. (#219663)
If a mul/shl has NSW and both operands are known-non-negative, add NUW.
Note the changes in loopflatten.ll. The IV is now widened by
LoopFlatten, converting it into a form the pass cannot flatten. This is
an underlying issue in the pass
This improves flags in a number of cases, leading to a number of
additional folds:
https://github.com/dtcxzyw/llvm-opt-benchmark-nightly/pull/1098
Alive2 Proof: https://alive2.llvm.org/ce/z/C7QZ4P
PR: https://github.com/llvm/llvm-project/pull/219663
Added:
Modified:
clang/test/CodeGen/attr-counted-by-or-null-for-pointers.c
clang/test/CodeGen/attr-sized-by-for-pointers.c
clang/test/CodeGen/attr-sized-by-or-null-for-pointers.c
llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
llvm/test/Transforms/ConstraintElimination/gep-arithmetic-different-types.ll
llvm/test/Transforms/ConstraintElimination/materialize-flags.ll
llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
Removed:
################################################################################
diff --git a/clang/test/CodeGen/attr-counted-by-or-null-for-pointers.c b/clang/test/CodeGen/attr-counted-by-or-null-for-pointers.c
index fcb70bd371337..56b52fae804d2 100644
--- a/clang/test/CodeGen/attr-counted-by-or-null-for-pointers.c
+++ b/clang/test/CodeGen/attr-counted-by-or-null-for-pointers.c
@@ -136,7 +136,7 @@ size_t test_bdos_of_pointer_through_cast(struct annotated_ptr *p, int index) {
// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP4]], null
// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[COUNT]], 3
// SANITIZE-WITH-ATTR-NEXT: [[ARRAY_SIZE:%.*]] = select i1 [[TMP5]], i64 0, i64 [[TMP6]]
-// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nsw i64 [[IDXPROM]], 3
+// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nuw nsw i64 [[IDXPROM]], 3
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = sub nsw i64 [[ARRAY_SIZE]], [[INDEX_SIZE]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP7:%.*]] = tail call i64 @llvm.smax.i64(i64 [[RESULT]], i64 0)
// SANITIZE-WITH-ATTR-NEXT: ret i64 [[TMP7]]
@@ -391,7 +391,6 @@ void test_pass_dynamic_object_size_of_pointer(struct annotated_volatile_ptr *p)
// The '&((cast) ptr->array)[idx]' form via pass_dynamic_object_size.
// FIXME: for the _or_null variants the object-size null-check re-emits the
// `ptr->array` load. Instead it should reuse the value.
-//
// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test_pass_dynamic_object_size_of_element_address(
// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
// SANITIZE-WITH-ATTR-NEXT: [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/attr-sized-by-for-pointers.c b/clang/test/CodeGen/attr-sized-by-for-pointers.c
index 709e2ac2af2af..8bf77bcf71ca5 100644
--- a/clang/test/CodeGen/attr-sized-by-for-pointers.c
+++ b/clang/test/CodeGen/attr-sized-by-for-pointers.c
@@ -198,7 +198,7 @@ size_t test_bdos_of_pointer_multibyte_pointee(struct annotated_sized_ptr_non_byt
// SANITIZE-WITH-ATTR: [[CONT8]]:
// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: [[COUNT:%.*]] = zext nneg i32 [[DOTCOUNTED_BY_LOAD]] to i64
-// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nsw i64 [[IDXPROM]], 3
+// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nuw nsw i64 [[IDXPROM]], 3
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = sub nsw i64 [[COUNT]], [[INDEX_SIZE]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.smax.i64(i64 [[RESULT]], i64 0)
// SANITIZE-WITH-ATTR-NEXT: ret i64 [[TMP4]]
@@ -487,7 +487,7 @@ void test_pass_dynamic_object_size_of_element_address(struct annotated_volatile_
// SANITIZE-WITH-ATTR: [[CONT8]]:
// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: [[COUNT:%.*]] = zext nneg i32 [[DOTCOUNTED_BY_LOAD]] to i64
-// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nsw i64 [[IDXPROM]], 2
+// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nuw nsw i64 [[IDXPROM]], 2
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = sub nsw i64 [[COUNT]], [[INDEX_SIZE]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.smax.i64(i64 [[RESULT]], i64 0)
// SANITIZE-WITH-ATTR-NEXT: ret i64 [[TMP4]]
diff --git a/clang/test/CodeGen/attr-sized-by-or-null-for-pointers.c b/clang/test/CodeGen/attr-sized-by-or-null-for-pointers.c
index 5f673e44a8ec3..61ca39b37d5f2 100644
--- a/clang/test/CodeGen/attr-sized-by-or-null-for-pointers.c
+++ b/clang/test/CodeGen/attr-sized-by-or-null-for-pointers.c
@@ -234,7 +234,7 @@ size_t test_bdos_of_pointer_multibyte_pointee(struct annotated_sized_ptr_non_byt
// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP4]], null
// SANITIZE-WITH-ATTR-NEXT: [[COUNT_OR_NULL:%.*]] = select i1 [[TMP5]], i64 0, i64 [[COUNT]]
-// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nsw i64 [[IDXPROM]], 3
+// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nuw nsw i64 [[IDXPROM]], 3
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = sub nsw i64 [[COUNT_OR_NULL]], [[INDEX_SIZE]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = tail call i64 @llvm.smax.i64(i64 [[RESULT]], i64 0)
// SANITIZE-WITH-ATTR-NEXT: ret i64 [[TMP6]]
@@ -563,7 +563,7 @@ void test_pass_dynamic_object_size_of_element_address(struct annotated_volatile_
// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP4]], null
// SANITIZE-WITH-ATTR-NEXT: [[COUNT_OR_NULL:%.*]] = select i1 [[TMP5]], i64 0, i64 [[COUNT]]
-// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nsw i64 [[IDXPROM]], 2
+// SANITIZE-WITH-ATTR-NEXT: [[INDEX_SIZE:%.*]] = shl nuw nsw i64 [[IDXPROM]], 2
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = sub nsw i64 [[COUNT_OR_NULL]], [[INDEX_SIZE]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = tail call i64 @llvm.smax.i64(i64 [[RESULT]], i64 0)
// SANITIZE-WITH-ATTR-NEXT: ret i64 [[TMP6]]
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 65cfcd78205f6..ef324c40e44a6 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1307,9 +1307,10 @@ static bool canStrengthenFlags(Instruction *I) {
case Instruction::Shl:
if (BO->hasNoUnsignedWrap() && BO->hasNoSignedWrap())
return false;
- // A constant second operand can be used to bound the first operand to
- // refine no-wrap flags.
- return isa<ConstantInt>(BO->getOperand(1));
+ // With a constant second operand, we can use bounds on the first operand to
+ // refine no-wrap flags. Independently, nuw can be added for nsw if the
+ // operands are non-negative.
+ return isa<ConstantInt>(BO->getOperand(1)) || BO->hasNoSignedWrap();
default:
return false;
}
@@ -1367,26 +1368,34 @@ static bool tryToStrengthenFlags(Instruction *I, ConstraintInfo &Info,
bool Changed = false;
// For a constant Op1, the ranges of Op0 for which the operation does not
// wrap are known exactly; check if the systems imply one of them.
- auto *C = cast<ConstantInt>(Op1);
- ConstantRange Other(C->getValue());
- if (!I->hasNoUnsignedWrap() &&
- doesHoldInRange(Info, Op0,
- ConstantRange::makeGuaranteedNoWrapRegion(
- Opcode, Other, OBO::NoUnsignedWrap),
- /*Signed=*/false)) {
+ if (auto *C = dyn_cast<ConstantInt>(Op1)) {
+ ConstantRange Other(C->getValue());
+ if (!I->hasNoUnsignedWrap() &&
+ doesHoldInRange(Info, Op0,
+ ConstantRange::makeGuaranteedNoWrapRegion(
+ Opcode, Other, OBO::NoUnsignedWrap),
+ /*Signed=*/false)) {
+ LLVM_DEBUG(dbgs() << "Adding nuw to " << *I << "\n");
+ I->setHasNoUnsignedWrap();
+ Changed = true;
+ }
+ if (!I->hasNoSignedWrap() &&
+ doesHoldInRange(Info, Op0,
+ ConstantRange::makeGuaranteedNoWrapRegion(
+ Opcode, Other, OBO::NoSignedWrap),
+ /*Signed=*/true)) {
+ LLVM_DEBUG(dbgs() << "Adding nsw to " << *I << "\n");
+ I->setHasNoSignedWrap();
+ Changed = true;
+ }
+ }
+ if (!I->hasNoUnsignedWrap() && I->hasNoSignedWrap() &&
+ Info.isKnownNonNegative(Op0) &&
+ (Opcode == Instruction::Shl || Info.isKnownNonNegative(Op1))) {
LLVM_DEBUG(dbgs() << "Adding nuw to " << *I << "\n");
I->setHasNoUnsignedWrap();
Changed = true;
}
- if (!I->hasNoSignedWrap() &&
- doesHoldInRange(Info, Op0,
- ConstantRange::makeGuaranteedNoWrapRegion(
- Opcode, Other, OBO::NoSignedWrap),
- /*Signed=*/true)) {
- LLVM_DEBUG(dbgs() << "Adding nsw to " << *I << "\n");
- I->setHasNoSignedWrap();
- Changed = true;
- }
return Changed;
}
default:
diff --git a/llvm/test/Transforms/ConstraintElimination/gep-arithmetic-
diff erent-types.ll b/llvm/test/Transforms/ConstraintElimination/gep-arithmetic-
diff erent-types.ll
index fe67e0a822d9a..47d8ec22dda42 100644
--- a/llvm/test/Transforms/ConstraintElimination/gep-arithmetic-
diff erent-types.ll
+++ b/llvm/test/Transforms/ConstraintElimination/gep-arithmetic-
diff erent-types.ll
@@ -123,11 +123,11 @@ define i1 @gep_shl_nsw_positive_index(ptr %A, ptr %upper, i8 %idx) {
; CHECK-LABEL: @gep_shl_nsw_positive_index(
; CHECK-NEXT: [[IDX_POS:%.*]] = icmp sge i8 [[IDX:%.*]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[IDX_POS]])
-; CHECK-NEXT: [[IDX_2:%.*]] = shl nsw i8 [[IDX]], 2
+; CHECK-NEXT: [[IDX_2:%.*]] = shl nuw nsw i8 [[IDX]], 2
; CHECK-NEXT: [[ADD_I8_IDX_2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i8 [[IDX_2]]
; CHECK-NEXT: [[C_0:%.*]] = icmp ult ptr [[ADD_I8_IDX_2]], [[UPPER:%.*]]
; CHECK-NEXT: call void @llvm.assume(i1 [[C_0]])
-; CHECK-NEXT: [[IDX_1:%.*]] = shl nsw i8 [[IDX]], 1
+; CHECK-NEXT: [[IDX_1:%.*]] = shl nuw nsw i8 [[IDX]], 1
; CHECK-NEXT: [[ADD_I32_IDX_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i8 [[IDX_1]]
; CHECK-NEXT: [[C_1:%.*]] = icmp ult ptr [[ADD_I32_IDX_1]], [[UPPER]]
; CHECK-NEXT: [[ADD_I8_IDX_1:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[IDX_1]]
diff --git a/llvm/test/Transforms/ConstraintElimination/materialize-flags.ll b/llvm/test/Transforms/ConstraintElimination/materialize-flags.ll
index 2ec78db674b04..ef858b2252639 100644
--- a/llvm/test/Transforms/ConstraintElimination/materialize-flags.ll
+++ b/llvm/test/Transforms/ConstraintElimination/materialize-flags.ll
@@ -541,7 +541,7 @@ define i64 @mul_nuw_from_nsw_and_non_negative(i64 %x, i64 %y, i64 %z) {
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: [[C_2:%.*]] = icmp sge i64 [[Y]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[X]], [[Y]]
+; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i64 [[X]], [[Y]]
; CHECK-NEXT: ret i64 [[MUL]]
;
entry:
@@ -585,7 +585,7 @@ define i64 @shl_nuw_from_nsw_and_non_negative(i64 %x, i64 %s, i64 %z) {
; CHECK-NEXT: call void @llvm.assume(i1 [[C_0]])
; CHECK-NEXT: [[C_1:%.*]] = icmp sge i64 [[Z]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
-; CHECK-NEXT: [[SHL:%.*]] = shl nsw i64 [[X]], [[S]]
+; CHECK-NEXT: [[SHL:%.*]] = shl nuw nsw i64 [[X]], [[S]]
; CHECK-NEXT: ret i64 [[SHL]]
;
entry:
diff --git a/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll b/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
index 4bae51d2fcb04..3bb135c957574 100644
--- a/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
+++ b/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
@@ -16,11 +16,11 @@ define void @slt_mul_nsw_3_known_positive_1(i8 %start, i8 %high) {
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[START_1:%.*]] = mul nuw nsw i8 [[START]], 1
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT: [[START_2:%.*]] = mul nuw nsw i8 [[START]], 2
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT: [[START_3:%.*]] = mul nuw nsw i8 [[START]], 3
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT: [[START_4:%.*]] = mul nuw nsw i8 [[START]], 4
; CHECK-NEXT: [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_3]])
; CHECK-NEXT: ret void
@@ -124,13 +124,13 @@ define void @slt_mul_no_nsw_3_known_positive_1(i8 %start, i8 %high) {
; CHECK-NEXT: [[START_1:%.*]] = mul nuw nsw i8 [[START]], 1
; CHECK-NEXT: [[C_4:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_4]])
-; CHECK-NEXT: [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT: [[START_2:%.*]] = mul nuw nsw i8 [[START]], 2
; CHECK-NEXT: [[C_5:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_5]])
-; CHECK-NEXT: [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT: [[START_3:%.*]] = mul nuw nsw i8 [[START]], 3
; CHECK-NEXT: [[C_6:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_6]])
-; CHECK-NEXT: [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT: [[START_4:%.*]] = mul nuw nsw i8 [[START]], 4
; CHECK-NEXT: [[C_7:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_7]])
; CHECK-NEXT: ret void
@@ -656,11 +656,11 @@ define void @slt_mul_nsw_3_known_nonnegative_1(i8 %start, i8 %high) {
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[START_1:%.*]] = mul nuw nsw i8 [[START]], 1
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT: [[START_2:%.*]] = mul nuw nsw i8 [[START]], 2
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT: [[START_3:%.*]] = mul nuw nsw i8 [[START]], 3
; CHECK-NEXT: call void @use(i1 true)
-; CHECK-NEXT: [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT: [[START_4:%.*]] = mul nuw nsw i8 [[START]], 4
; CHECK-NEXT: [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_3]])
; CHECK-NEXT: ret void
@@ -768,13 +768,13 @@ define void @slt_mul_nsw_both_var_non_negative_1(i8 %start, i8 %scale, i8 %high)
; CHECK-NEXT: [[START_1:%.*]] = mul nuw nsw i8 [[START]], 1
; CHECK-NEXT: [[T_1:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[T_1]])
-; CHECK-NEXT: [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT: [[START_2:%.*]] = mul nuw nsw i8 [[START]], 2
; CHECK-NEXT: [[T_2:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[T_2]])
-; CHECK-NEXT: [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT: [[START_3:%.*]] = mul nuw nsw i8 [[START]], 3
; CHECK-NEXT: [[T_3:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[T_3]])
-; CHECK-NEXT: [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT: [[START_4:%.*]] = mul nuw nsw i8 [[START]], 4
; CHECK-NEXT: [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
; CHECK-NEXT: call void @use(i1 [[C_3]])
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
index 8f106d720f758..76e328329ea74 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
@@ -1,64 +1,155 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes="default<O3>" -enable-loop-flatten -loop-flatten-cost-threshold=3 -S %s | FileCheck %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"
-define dso_local void @_Z3fooPiii(ptr %A, i32 %N, i32 %M) #0 {
-; CHECK-LABEL: @_Z3fooPiii(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[M:%.*]], 0
+; TODO: @foo_32_bit_iv is no longer flattened, because its i32 bit IV gets
+; widened and LoopFlatten does not recognize the wide form.
+define void @foo_32_bit_iv(ptr %A, i32 %N, i32 %M) {
+; CHECK-LABEL: define void @foo_32_bit_iv(
+; CHECK-SAME: ptr nofree readonly captures(none) [[A:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[M]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[CMP3]], i1 [[CMP21]], i1 false
-; CHECK-NEXT: br i1 [[OR_COND]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.cond1.preheader.lr.ph.split:
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[INNER_HEADER_PREHEADER_LR_PH_SPLIT:.*]], label %[[EXIT:.*]]
+; CHECK: [[INNER_HEADER_PREHEADER_LR_PH_SPLIT]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[M]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[N]] to i64
-; CHECK-NEXT: [[FLATTEN_TRIPCOUNT:%.*]] = mul nuw nsw i64 [[TMP0]], [[TMP1]]
-; CHECK-NEXT: br label [[FOR_COND1_PREHEADER_US:%.*]]
-; CHECK: for.cond1.preheader:
-; CHECK-NEXT: [[INDVAR6:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US]] ], [ [[INDVAR_NEXT7:%.*]], [[FOR_COND1_PREHEADER_US]] ]
-; CHECK-NEXT: [[ARRAYIDX_US:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A:%.*]], i64 [[INDVAR6]]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
+; CHECK-NEXT: br label %[[INNER_HEADER_PREHEADER:.*]]
+; CHECK: [[INNER_HEADER_PREHEADER]]:
+; CHECK-NEXT: [[INDVAR6:%.*]] = phi i64 [ 0, %[[INNER_HEADER_PREHEADER_LR_PH_SPLIT]] ], [ [[INDVAR_NEXT7:%.*]], %[[INNER_HEADER_OUTER_LATCH_CRIT_EDGE:.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[INDVAR6]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[M]], [[TMP5]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT: br label %[[INNER_LATCH:.*]]
+; CHECK: [[INNER_LATCH]]:
+; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ 0, %[[INNER_HEADER_PREHEADER]] ], [ [[INDVAR_NEXT:%.*]], %[[INNER_LATCH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr [4 x i8], ptr [[INVARIANT_GEP]], i64 [[INDVAR]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: tail call void @_Z1fi(i32 [[TMP2]])
-; CHECK-NEXT: [[INDVAR_NEXT7]] = add nuw i64 [[INDVAR6]], 1
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR_NEXT7]], [[FLATTEN_TRIPCOUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_COND1_PREHEADER_US]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[INDVAR_NEXT]] = add nuw nsw i64 [[INDVAR]], 1
+; CHECK-NEXT: [[EXITCOND_NOT1:%.*]] = icmp eq i64 [[INDVAR_NEXT]], [[TMP0]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT1]], label %[[INNER_HEADER_OUTER_LATCH_CRIT_EDGE]], label %[[INNER_LATCH]]
+; CHECK: [[INNER_HEADER_OUTER_LATCH_CRIT_EDGE]]:
+; CHECK-NEXT: [[INDVAR_NEXT7]] = add nuw nsw i64 [[INDVAR6]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR_NEXT7]], [[TMP1]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[INNER_HEADER_PREHEADER]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
- br label %for.cond
+ br label %outer.header
-for.cond:
- %i.0 = phi i32 [ 0, %entry ], [ %inc6, %for.cond.cleanup3 ]
- %cmp = icmp slt i32 %i.0, %N
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+outer.header:
+ %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %cmp = icmp slt i32 %outer.iv, %N
+ br i1 %cmp, label %outer.body, label %exit
-for.cond.cleanup:
- ret void
-
-for.body:
- br label %for.cond1
-
-for.cond1:
- %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.body4 ]
- %cmp2 = icmp slt i32 %j.0, %M
- br i1 %cmp2, label %for.body4, label %for.cond.cleanup3
+outer.body:
+ br label %inner.header
-for.cond.cleanup3:
- %inc6 = add nsw i32 %i.0, 1
- br label %for.cond
+inner.header:
+ %inner.iv = phi i32 [ 0, %outer.body ], [ %inner.iv.next, %inner.latch ]
+ %cmp2 = icmp slt i32 %inner.iv, %M
+ br i1 %cmp2, label %inner.latch, label %outer.latch
-for.body4:
- %mul = mul nsw i32 %i.0, %M
- %add = add nsw i32 %mul, %j.0
+inner.latch:
+ %mul = mul nsw i32 %outer.iv, %M
+ %add = add nsw i32 %mul, %inner.iv
%idxprom = sext i32 %add to i64
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom
%0 = load i32, ptr %arrayidx, align 4
call void @_Z1fi(i32 %0)
- %inc = add nsw i32 %j.0, 1
- br label %for.cond1
+ %inner.iv.next = add nsw i32 %inner.iv, 1
+ br label %inner.header
+
+outer.latch:
+ %outer.iv.next = add nsw i32 %outer.iv, 1
+ br label %outer.header
+
+exit:
+ ret void
}
declare dso_local void @_Z1fi(i32) #2
+
+; Same nest, but with i64 induction variables.
+define dso_local void @foo_64_bit_iv(ptr %A, i64 %N, i64 %M) {
+; CHECK-LABEL: define dso_local void @foo_64_bit_iv(
+; CHECK-SAME: ptr nofree readonly captures(none) [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: [[CMP21:%.*]] = icmp sgt i64 [[M]], 0
+; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[CMP3]], i1 [[CMP21]], i1 false
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[INNER_HEADER_PREHEADER_LVER_CHECK:.*]], label %[[EXIT:.*]]
+; CHECK: [[INNER_HEADER_PREHEADER_LVER_CHECK]]:
+; CHECK-NEXT: [[FLATTEN_MUL:%.*]] = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[N]], i64 [[M]])
+; CHECK-NEXT: [[FLATTEN_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[FLATTEN_MUL]], 1
+; CHECK-NEXT: br i1 [[FLATTEN_OVERFLOW]], label %[[INNER_HEADER_PREHEADER_LVER_ORIG:.*]], label %[[INNER_HEADER_PREHEADER_PREHEADER:.*]]
+; CHECK: [[INNER_HEADER_PREHEADER_PREHEADER]]:
+; CHECK-NEXT: [[FLATTEN_TRIPCOUNT:%.*]] = extractvalue { i64, i1 } [[FLATTEN_MUL]], 0
+; CHECK-NEXT: [[SMAX:%.*]] = tail call i64 @llvm.smax.i64(i64 [[FLATTEN_TRIPCOUNT]], i64 1)
+; CHECK-NEXT: br label %[[INNER_HEADER_PREHEADER:.*]]
+; CHECK: [[INNER_HEADER_PREHEADER_LVER_ORIG]]:
+; CHECK-NEXT: [[OUTER_IV4_LVER_ORIG:%.*]] = phi i64 [ [[OUTER_IV_NEXT_LVER_ORIG:%.*]], %[[INNER_HEADER_OUTER_LATCH_CRIT_EDGE_LVER_ORIG:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER_LVER_CHECK]] ]
+; CHECK-NEXT: [[MUL_LVER_ORIG:%.*]] = mul nuw nsw i64 [[OUTER_IV4_LVER_ORIG]], [[M]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[MUL_LVER_ORIG]]
+; CHECK-NEXT: br label %[[INNER_LATCH_LVER_ORIG:.*]]
+; CHECK: [[INNER_LATCH_LVER_ORIG]]:
+; CHECK-NEXT: [[INNER_IV2_LVER_ORIG:%.*]] = phi i64 [ 0, %[[INNER_HEADER_PREHEADER_LVER_ORIG]] ], [ [[INNER_IV_NEXT_LVER_ORIG:%.*]], %[[INNER_LATCH_LVER_ORIG]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr [4 x i8], ptr [[TMP0]], i64 [[INNER_IV2_LVER_ORIG]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4
+; CHECK-NEXT: tail call void @_Z1fi(i32 [[TMP1]])
+; CHECK-NEXT: [[INNER_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INNER_IV2_LVER_ORIG]], 1
+; CHECK-NEXT: [[EXITCOND7_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT_LVER_ORIG]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND7_NOT]], label %[[INNER_HEADER_OUTER_LATCH_CRIT_EDGE_LVER_ORIG]], label %[[INNER_LATCH_LVER_ORIG]]
+; CHECK: [[INNER_HEADER_OUTER_LATCH_CRIT_EDGE_LVER_ORIG]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[OUTER_IV4_LVER_ORIG]], 1
+; CHECK-NEXT: [[EXITCOND8_NOT:%.*]] = icmp eq i64 [[OUTER_IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND8_NOT]], label %[[EXIT]], label %[[INNER_HEADER_PREHEADER_LVER_ORIG]]
+; CHECK: [[INNER_HEADER_PREHEADER]]:
+; CHECK-NEXT: [[OUTER_IV4:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], %[[INNER_HEADER_PREHEADER]] ], [ 0, %[[INNER_HEADER_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: [[FLATTEN_ARRAYIDX:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OUTER_IV4]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[FLATTEN_ARRAYIDX]], align 4
+; CHECK-NEXT: tail call void @_Z1fi(i32 [[TMP2]])
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV4]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[SMAX]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[INNER_HEADER_PREHEADER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %cmp = icmp slt i64 %outer.iv, %N
+ br i1 %cmp, label %outer.body, label %exit
+
+outer.body:
+ br label %inner.header
+
+inner.header:
+ %inner.iv = phi i64 [ 0, %outer.body ], [ %inner.iv.next, %inner.latch ]
+ %cmp2 = icmp slt i64 %inner.iv, %M
+ br i1 %cmp2, label %inner.latch, label %outer.latch
+
+inner.latch:
+ %mul = mul nsw i64 %outer.iv, %M
+ %add = add nsw i64 %mul, %inner.iv
+ %arrayidx = getelementptr inbounds i32, ptr %A, i64 %add
+ %0 = load i32, ptr %arrayidx, align 4
+ call void @_Z1fi(i32 %0)
+ %inner.iv.next = add nsw i64 %inner.iv, 1
+ br label %inner.header
+
+outer.latch:
+ %outer.iv.next = add nsw i64 %outer.iv, 1
+ br label %outer.header
+
+exit:
+ ret void
+}
More information about the cfe-commits
mailing list