[llvm] [AArch64] Don't add runtime SCEV checks from containsDecreasingPointers. (PR #216961)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 02:20:53 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
containsDecreasingPointers is a heuristic query, called from preferPredicateOverEpilogue to decide whether tail-folding needs TailFoldingOpts::Reverse. It only guides tail-folding decision, but permanently added predicates, which may not even be used later.
Request the predicates explicitly and discard them instead. They are not needed for correctness; the vectorizer has to add all predicates needed for correctness independent of TTI.
---
Full diff: https://github.com/llvm/llvm-project/pull/216961.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+5-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll (+9-61)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 2e28901cc5127..21b13e4b107c4 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -7226,8 +7226,12 @@ static bool containsDecreasingPointers(Loop *TheLoop,
if (isa<LoadInst>(&I) || isa<StoreInst>(&I)) {
Value *Ptr = getLoadStorePointerOperand(&I);
Type *AccessTy = getLoadStoreType(&I);
+ // Analyze assuming predicates will be added, but discard them; this
+ // query only guides tail-folding and must not add runtime checks to the
+ // loop.
+ SmallVector<const SCEVPredicate *> Predicates;
if (getPtrStride(*PSE, AccessTy, Ptr, TheLoop, DT, Strides,
- /*Assume=*/true, /*ShouldCheckWrap=*/false)
+ /*ShouldCheckWrap=*/false, &Predicates)
.value_or(0) < 0)
return true;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 898a5d9e8a461..8a56d86a2e6a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -203,24 +203,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[MUL_X:%.*]] = add i32 [[X]], 1
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; DEFAULT: [[VECTOR_SCEVCHECK]]:
-; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 -1, [[X]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = icmp slt i32 [[MUL_X]], 0
-; DEFAULT-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[MUL_X]]
-; DEFAULT-NEXT: [[TMP4:%.*]] = trunc i64 [[N]] to i32
-; DEFAULT-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 [[TMP4]])
-; DEFAULT-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
-; DEFAULT-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
-; DEFAULT-NEXT: [[TMP5:%.*]] = sub i32 0, [[MUL_RESULT]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[TMP5]], 0
-; DEFAULT-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 false
-; DEFAULT-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[N]], 4294967295
-; DEFAULT-NEXT: [[TMP10:%.*]] = icmp ne i32 [[MUL_X]], 0
-; DEFAULT-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]]
-; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
-; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -254,7 +237,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
; DEFAULT: [[FOR_BODY]]:
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -314,25 +297,7 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[MUL:%.*]] = mul i32 [[X]], [[X]]
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; DEFAULT: [[VECTOR_SCEVCHECK]]:
-; DEFAULT-NEXT: [[TMP1:%.*]] = mul i32 [[X]], [[X]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = sub i32 0, [[TMP1]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = icmp slt i32 [[MUL]], 0
-; DEFAULT-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 [[MUL]]
-; DEFAULT-NEXT: [[TMP5:%.*]] = trunc i64 [[N]] to i32
-; DEFAULT-NEXT: [[MUL1:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP4]], i32 [[TMP5]])
-; DEFAULT-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL1]], 0
-; DEFAULT-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL1]], 1
-; DEFAULT-NEXT: [[TMP6:%.*]] = sub i32 0, [[MUL_RESULT]]
-; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], 0
-; DEFAULT-NEXT: [[TMP8:%.*]] = select i1 [[TMP3]], i1 [[TMP7]], i1 false
-; DEFAULT-NEXT: [[TMP9:%.*]] = or i1 [[TMP8]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[TMP10:%.*]] = icmp ugt i64 [[N]], 4294967295
-; DEFAULT-NEXT: [[TMP11:%.*]] = icmp ne i32 [[MUL]], 0
-; DEFAULT-NEXT: [[TMP12:%.*]] = and i1 [[TMP10]], [[TMP11]]
-; DEFAULT-NEXT: [[TMP13:%.*]] = or i1 [[TMP9]], [[TMP12]]
-; DEFAULT-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -370,12 +335,12 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[IV_1_TRUNC:%.*]] = trunc i64 [[IV_1]] to i32
; DEFAULT-NEXT: [[IV_1_MUL:%.*]] = mul i32 [[MUL]], [[IV_1_TRUNC]]
; DEFAULT-NEXT: [[IV_2_NEXT]] = add i32 [[IV_2]], 1
@@ -436,24 +401,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[ADD:%.*]] = add i32 [[X]], 1
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; DEFAULT: [[VECTOR_SCEVCHECK]]:
-; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 -1, [[X]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = icmp slt i32 [[ADD]], 0
-; DEFAULT-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[ADD]]
-; DEFAULT-NEXT: [[TMP4:%.*]] = trunc i64 [[N]] to i32
-; DEFAULT-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 [[TMP4]])
-; DEFAULT-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
-; DEFAULT-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
-; DEFAULT-NEXT: [[TMP5:%.*]] = sub i32 0, [[MUL_RESULT]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[TMP5]], 0
-; DEFAULT-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 false
-; DEFAULT-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[N]], 4294967295
-; DEFAULT-NEXT: [[TMP10:%.*]] = icmp ne i32 [[ADD]], 0
-; DEFAULT-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]]
-; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
-; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -491,8 +439,8 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
``````````
</details>
https://github.com/llvm/llvm-project/pull/216961
More information about the llvm-commits
mailing list