[llvm] [InstSimplify] Defer to ConstantFold in simplifyInstrinsic (PR #205061)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 03:44:59 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/205061
>From 9ca53c365b07d879ece0e7428b579df567b36b44 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 10 Jun 2026 12:26:52 +0100
Subject: [PATCH 1/6] [InstSimplifyFolder] Fallback to TargetFold on intrinsics
The InstSimplifyFolder is not strictly more powerful than the
TargetFolder in the case of intrinsics. Fix this by adding a fallback.
---
.../llvm/Analysis/InstSimplifyFolder.h | 2 +
.../truncate-to-minimal-bitwidth-cost.ll | 15 +------
.../Transforms/LoopVectorize/X86/funclet.ll | 3 +-
.../LoopVectorize/constantfolder.ll | 39 ++++++++++++++++++-
...aved-accesses-requiring-scev-predicates.ll | 33 +++-------------
5 files changed, 47 insertions(+), 45 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 16286b93677f8..4fd355801f2c1 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -123,6 +123,8 @@ class LLVM_ABI InstSimplifyFolder final : public IRBuilderFolder {
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
FastMathFlags FMF = {},
Function *CtxF = nullptr) const override {
+ if (Value *V = ConstFolder.FoldIntrinsic(ID, Ops, Ty, FMF))
+ return V;
return simplifyIntrinsic(ID, Ty, Ops, FMF, SQ, CtxF);
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 30ea84b642489..a6072f1217727 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -253,20 +253,9 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 4, i16 255)
-; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = add i16 4, [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i16 [[TMP0]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP1]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 255)
-; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW3]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP2]], [[TMP5]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[P2]], i64 1021
diff --git a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
index 58812b5e2309d..fa858d6d6fbbc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
@@ -26,10 +26,9 @@ define void @test1() #0 personality ptr @__CxxFrameHandler3 {
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[TRY_CONT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[TRY_CONT]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x double> [[TMP2]], i64 15
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: store double [[TMP4]], ptr @sink, align 8
+; CHECK-NEXT: store double 1.000000e+00, ptr @sink, align 8
; CHECK-NEXT: catchret from [[TMP1]] to label %[[TRY_CONT1:.*]]
; CHECK: [[TRY_CONT1]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index a2b5628a84f68..1577a48727393 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -323,6 +323,41 @@ exit:
ret void
}
+define void @const_fold_unaryintrinsic(ptr %p) {
+; CHECK-LABEL: define void @const_fold_unaryintrinsic(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr double, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x double> splat (double 1.000000e+00), ptr [[TMP0]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.p.iv = getelementptr double, ptr %p, i64 %iv
+ %res = call double @llvm.canonicalize.f64(double 1.0)
+ store double %res, ptr %gep.p.iv
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
; CHECK-LABEL: define void @const_fold_widegep(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[D:%.*]]) {
@@ -335,7 +370,7 @@ define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
; CHECK-NEXT: store ptr [[A]], ptr [[B]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -432,7 +467,7 @@ define void @unused_live_ins_in_preheader(ptr %dst, ptr %src) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
index 94bc48a726316..a869a177aca21 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
@@ -122,33 +122,10 @@ exit:
define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr noalias %x, ptr noalias %out) {
; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(
; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[START:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[MUL:%.*]] = call { i4, i1 } @llvm.umul.with.overflow.i4(i4 5, i4 -1)
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i4, i1 } [[MUL]], 1
-; CHECK-NEXT: br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 15
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP1]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <20 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <20 x i32> [[WIDE_VEC]], <20 x i32> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP3]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 12, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 5
; CHECK-NEXT: [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15
@@ -157,7 +134,7 @@ define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr
; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 16
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -213,7 +190,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -229,7 +206,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 49d244c623c13e9328c86ac33cb9a722b8cbc09b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 10:24:29 +0100
Subject: [PATCH 2/6] [InstSimplify] Modify llvm::simplifyIntrinsic directly
---
llvm/include/llvm/Analysis/InstSimplifyFolder.h | 2 --
llvm/lib/Analysis/InstructionSimplify.cpp | 7 +++++++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 4fd355801f2c1..16286b93677f8 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -123,8 +123,6 @@ class LLVM_ABI InstSimplifyFolder final : public IRBuilderFolder {
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
FastMathFlags FMF = {},
Function *CtxF = nullptr) const override {
- if (Value *V = ConstFolder.FoldIntrinsic(ID, Ops, Ty, FMF))
- return V;
return simplifyIntrinsic(ID, Ty, Ops, FMF, SQ, CtxF);
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index fcd7f68260856..6498f19375610 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7289,6 +7289,13 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
any_of(Args, IsaPred<PoisonValue>))
return PoisonValue::get(ReturnType);
+ // Defer to ConstantFolding if all args are constants.
+ if (all_of(Args, IsaPred<Constant>))
+ if (Constant *C = ConstantFoldIntrinsic(
+ IID, ArrayRef((Constant *const *)Args.data(), Args.size()),
+ ReturnType))
+ return C;
+
// Most of the intrinsics with no operands have some kind of side effect.
// Don't simplify.
if (!NumOperands) {
>From 51f682bd8c234b6caf67e1ee829ba89b302dba59 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 12:04:13 +0100
Subject: [PATCH 3/6] [ConstFold] Fix the strictfp issue
There are a couple of unfortunate regressions, due to conservative
strictfp assumption.
---
llvm/include/llvm/Analysis/ConstantFolding.h | 6 ++-
llvm/lib/Analysis/ConstantFolding.cpp | 53 ++++++++++++-------
llvm/lib/Analysis/InstructionSimplify.cpp | 5 +-
.../SLPVectorizer/AArch64/uselistorder.ll | 6 ++-
.../SLPVectorizer/X86/reduce-constants.ll | 5 +-
5 files changed, 48 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 078f84369ca48..ad4c4c4ec5d7b 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -169,8 +169,12 @@ LLVM_ABI Constant *ConstantFoldCall(const CallBase *Call, Function *F,
const TargetLibraryInfo *TLI = nullptr,
bool AllowNonDeterministic = true);
+/// Attempt to constant fold an intrinsic given its operands \p Ops, and return
+/// type \p Ty. We also need to know whether the strictfp exception behavior is
+/// present, although \p IsStrictFP conservatively defaults to true.
LLVM_ABI Constant *ConstantFoldIntrinsic(Intrinsic::ID ID,
- ArrayRef<Constant *> Ops, Type *Ty);
+ ArrayRef<Constant *> Ops, Type *Ty,
+ bool IsStrictFP = true);
/// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
/// returning null if unsuccessful. Can cast pointer to pointer or pointer to
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 7fd1e60033437..67ca59802e22f 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1730,23 +1730,9 @@ Constant *llvm::ConstantFoldIntegerCast(Constant *C, Type *DestTy,
// Constant Folding for Calls
//
-bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
- if (Call->isNoBuiltin())
- return false;
- if (Call->getFunctionType() != F->getFunctionType())
- return false;
-
- // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
- // This can be useful for GPU targets or in cross-compilation scenarios
- // when the exact target FP behaviour is required, and the host compiler's
- // behaviour may be slightly different from the device's run-time behaviour.
- if (DisableFPCallFolding && (F->getReturnType()->isFloatingPointTy() ||
- any_of(F->args(), [](const Argument &Arg) {
- return Arg.getType()->isFloatingPointTy();
- })))
- return false;
-
- switch (F->getIntrinsicID()) {
+/// Returns true if the intrinsic can be constant folded, given \p IsStrictFP.
+static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
+ switch (ID) {
// Operations that do not operate floating-point numbers and do not depend on
// FP environment can be folded even in strictfp functions.
case Intrinsic::bswap:
@@ -2007,7 +1993,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
case Intrinsic::nvvm_sqrt_rn_d:
case Intrinsic::nvvm_sqrt_rn_f:
case Intrinsic::nvvm_sqrt_rn_ftz_f:
- return !Call->isStrictFP();
+ return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
case Intrinsic::nvvm_add_rm_d:
@@ -2104,8 +2090,27 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
return true;
default:
return false;
- case Intrinsic::not_intrinsic: break;
}
+}
+
+bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
+ if (Call->isNoBuiltin())
+ return false;
+ if (Call->getFunctionType() != F->getFunctionType())
+ return false;
+
+ // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
+ // This can be useful for GPU targets or in cross-compilation scenarios
+ // when the exact target FP behaviour is required, and the host compiler's
+ // behaviour may be slightly different from the device's run-time behaviour.
+ if (DisableFPCallFolding && (F->getReturnType()->isFloatingPointTy() ||
+ any_of(F->args(), [](const Argument &Arg) {
+ return Arg.getType()->isFloatingPointTy();
+ })))
+ return false;
+
+ if (F->getIntrinsicID() != Intrinsic::not_intrinsic)
+ return canConstantFoldIntrinsic(F->getIntrinsicID(), Call->isStrictFP());
if (!F->hasName() || Call->isStrictFP())
return false;
@@ -4705,7 +4710,15 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
} // end anonymous namespace
Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
- ArrayRef<Constant *> Ops, Type *Ty) {
+ ArrayRef<Constant *> Ops, Type *Ty,
+ bool IsStrictFP) {
+ if (DisableFPCallFolding &&
+ (Ty->isFloatingPointTy() || any_of(Ops, [](const Constant *Op) {
+ return Op->getType()->isFloatingPointTy();
+ })))
+ return nullptr;
+ if (!canConstantFoldIntrinsic(ID, IsStrictFP))
+ return nullptr;
return ConstantFoldScalarCall("", ID, Ty, Ops);
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 6498f19375610..f7d213dd58c2e 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7293,7 +7293,7 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
if (all_of(Args, IsaPred<Constant>))
if (Constant *C = ConstantFoldIntrinsic(
IID, ArrayRef((Constant *const *)Args.data(), Args.size()),
- ReturnType))
+ ReturnType, ExBehavior == fp::ebStrict))
return C;
// Most of the intrinsics with no operands have some kind of side effect.
@@ -7519,7 +7519,8 @@ static Value *simplifyIntrinsic(CallBase *Call, ArrayRef<Value *> Args,
}
default: {
// Use the default FP environment if none is found.
- fp::ExceptionBehavior ExBehavior = fp::ebIgnore;
+ fp::ExceptionBehavior ExBehavior =
+ Call->isStrictFP() ? fp::ebStrict : fp::ebIgnore;
RoundingMode Rounding = RoundingMode::NearestTiesToEven;
if (auto *Constrained = dyn_cast<ConstrainedFPIntrinsic>(Call)) {
ExBehavior = Constrained->getExceptionBehavior().value_or(ExBehavior);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
index 3a68a37c9f82c..e416addb1ae42 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
@@ -7,8 +7,10 @@ target triple = "aarch64-unknown-linux-gnu"
; This test has UB but the crash in #95016 only happens with it
define void @uselistorder_test() {
; CHECK-LABEL: @uselistorder_test(
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double 0.000000e+00, i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double 0.000000e+00, i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.maximum.f64(double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[TMP9:%.*]] = call double @llvm.maximum.f64(double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP10]], double [[TMP9]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> zeroinitializer, [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP4]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
index c91b9b0004028..793b9007667c7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
@@ -4,9 +4,10 @@
define double @test(double %m, double %a) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call fast double @llvm.maxnum.f64(double 0.000000e+00, double [[M:%.*]])
+; CHECK-NEXT: [[TMP0:%.*]] = call fast double @llvm.maxnum.f64(double 0.000000e+00, double 0.000000e+00)
; CHECK-NEXT: [[TMP1:%.*]] = call fast double @llvm.maxnum.f64(double [[TMP0]], double [[A:%.*]])
-; CHECK-NEXT: ret double [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.maxnum.f64(double [[TMP1]], double [[A1:%.*]])
+; CHECK-NEXT: ret double [[TMP2]]
;
entry:
%c = tail call fast double @llvm.maxnum.f64(double %m, double 0.000000e+00)
>From 16bc7d2b4454938c7d4c1e340b7537976959038b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 12:44:23 +0100
Subject: [PATCH 4/6] [IRBuilderFolder] Add param to fix regression
---
.../include/llvm/Analysis/InstSimplifyFolder.h | 7 ++++---
llvm/include/llvm/Analysis/TargetFolder.h | 7 ++++---
llvm/include/llvm/IR/ConstantFolder.h | 3 ++-
llvm/include/llvm/IR/IRBuilderFolder.h | 3 ++-
llvm/include/llvm/IR/NoFolder.h | 3 ++-
llvm/lib/IR/IRBuilder.cpp | 18 ++++++++++--------
.../InstCombine/InstCombineSelect.cpp | 9 ++++++---
.../SLPVectorizer/AArch64/uselistorder.ll | 6 ++----
.../SLPVectorizer/X86/reduce-constants.ll | 5 ++---
9 files changed, 34 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 16286b93677f8..a6591fafbd33e 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -121,9 +121,10 @@ class LLVM_ABI InstSimplifyFolder final : public IRBuilderFolder {
}
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
- FastMathFlags FMF = {},
- Function *CtxF = nullptr) const override {
- return simplifyIntrinsic(ID, Ty, Ops, FMF, SQ, CtxF);
+ FastMathFlags FMF = {}, Function *CtxF = nullptr,
+ bool IsStrictFP = true) const override {
+ return simplifyIntrinsic(ID, Ty, Ops, FMF, SQ, CtxF,
+ IsStrictFP ? fp::ebStrict : fp::ebIgnore);
}
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index ba6aca9106d43..8f61aa173fe79 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -192,11 +192,12 @@ class LLVM_ABI TargetFolder final : public IRBuilderFolder {
}
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
- FastMathFlags FMF = {},
- Function *CtxF = nullptr) const override {
+ FastMathFlags FMF = {}, Function *CtxF = nullptr,
+ bool IsStrictFP = true) const override {
if (all_of(Ops, IsaPred<Constant>))
return ConstantFoldIntrinsic(
- ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty);
+ ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty,
+ IsStrictFP);
return nullptr;
}
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index a3f56a0737022..06a59a34ae603 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -183,7 +183,8 @@ class LLVM_ABI ConstantFolder final : public IRBuilderFolder {
}
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
- FastMathFlags FMF, Function *CtxF) const override {
+ FastMathFlags FMF, Function *CtxF,
+ bool IsStrictFP) const override {
// Use TargetFolder or InstSimplifyFolder instead.
return nullptr;
}
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index 262d6c0d9e390..1d17bc636b4b3 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -78,7 +78,8 @@ class LLVM_ABI IRBuilderFolder {
virtual Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops,
Type *Ty, FastMathFlags FMF = {},
- Function *CtxF = nullptr) const = 0;
+ Function *CtxF = nullptr,
+ bool IsStrictFP = true) const = 0;
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 7b497053e3662..02f2f3d220d98 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -115,7 +115,8 @@ class LLVM_ABI NoFolder final : public IRBuilderFolder {
}
Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
- FastMathFlags FMF, Function *CtxF) const override {
+ FastMathFlags FMF, Function *CtxF,
+ bool IsStrictFP) const override {
return nullptr;
}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 3823ea2b08c83..d7da77de7bf45 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -923,9 +923,9 @@ Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
const Twine &Name) {
Module *M = BB->getModule();
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, Op->getType());
- if (Value *V =
- Folder.FoldIntrinsic(ID, Op, Fn->getReturnType(), FMFSource.get(FMF),
- GetInsertBlock()->getParent()))
+ if (Value *V = Folder.FoldIntrinsic(
+ ID, Op, Fn->getReturnType(), FMFSource.get(FMF),
+ GetInsertBlock()->getParent(), /*IsStrictFP=*/false))
return V;
return createCallHelper(Fn, Op, Name, FMFSource);
}
@@ -935,9 +935,9 @@ Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
const Twine &Name) {
Module *M = BB->getModule();
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {LHS->getType()});
- if (Value *V = Folder.FoldIntrinsic(ID, {LHS, RHS}, Fn->getReturnType(),
- FMFSource.get(FMF),
- GetInsertBlock()->getParent()))
+ if (Value *V = Folder.FoldIntrinsic(
+ ID, {LHS, RHS}, Fn->getReturnType(), FMFSource.get(FMF),
+ GetInsertBlock()->getParent(), /*IsStrictFP=*/false))
return V;
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
}
@@ -970,7 +970,8 @@ Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
function_ref<void(CallInst *)> SetFn) {
Type *RetTy = Intrinsic::getType(Context, ID, OverloadTypes)->getReturnType();
if (Value *V = Folder.FoldIntrinsic(ID, Args, RetTy, FMFSource.get(FMF),
- GetInsertBlock()->getParent()))
+ GetInsertBlock()->getParent(),
+ /*IsStrictFP=*/false))
return V;
CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
FMFSource, Name, OpBundles);
@@ -983,7 +984,8 @@ Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
FMFSource FMFSource, const Twine &Name,
function_ref<void(CallInst *)> SetFn) {
if (Value *V = Folder.FoldIntrinsic(ID, Args, RetTy, FMFSource.get(FMF),
- GetInsertBlock()->getParent()))
+ GetInsertBlock()->getParent(),
+ /*IsStrictFP=*/false))
return V;
CallInst *CI =
CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index f107c15304d9f..9b8935c159881 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -2271,10 +2271,12 @@ Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
return nullptr;
bool IsIntrinsic;
+ bool IsStrictFP;
unsigned Opcode;
if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(TrueVal)) {
Opcode = BOp->getOpcode();
IsIntrinsic = false;
+ IsStrictFP = false;
// This fold causes some regressions and is primarily intended for
// add and sub. So we early exit for div and rem to minimize the
@@ -2289,6 +2291,7 @@ Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
if (!match(II, m_MaxOrMin(m_Specific(X), m_Constant(C2))))
return nullptr;
Opcode = II->getIntrinsicID();
+ IsStrictFP = II->isStrictFP();
IsIntrinsic = true;
} else {
return nullptr;
@@ -2300,9 +2303,9 @@ Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
auto Flipped = getFlippedStrictnessPredicateAndConstant(Predicate, C1);
auto FoldBinaryOpOrIntrinsic = [&](Constant *LHS, Constant *RHS) {
- return IsIntrinsic
- ? ConstantFoldIntrinsic(Opcode, {LHS, RHS}, LHS->getType())
- : ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
+ return IsIntrinsic ? ConstantFoldIntrinsic(Opcode, {LHS, RHS},
+ LHS->getType(), IsStrictFP)
+ : ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
};
if (C3 == FoldBinaryOpOrIntrinsic(C1, C2)) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
index e416addb1ae42..3a68a37c9f82c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/uselistorder.ll
@@ -7,10 +7,8 @@ target triple = "aarch64-unknown-linux-gnu"
; This test has UB but the crash in #95016 only happens with it
define void @uselistorder_test() {
; CHECK-LABEL: @uselistorder_test(
-; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.maximum.f64(double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[TMP9:%.*]] = call double @llvm.maximum.f64(double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP10]], double [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double 0.000000e+00, i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double 0.000000e+00, i32 1
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> zeroinitializer, [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP4]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
index 793b9007667c7..c91b9b0004028 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduce-constants.ll
@@ -4,10 +4,9 @@
define double @test(double %m, double %a) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call fast double @llvm.maxnum.f64(double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[TMP0:%.*]] = call fast double @llvm.maxnum.f64(double 0.000000e+00, double [[M:%.*]])
; CHECK-NEXT: [[TMP1:%.*]] = call fast double @llvm.maxnum.f64(double [[TMP0]], double [[A:%.*]])
-; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.maxnum.f64(double [[TMP1]], double [[A1:%.*]])
-; CHECK-NEXT: ret double [[TMP2]]
+; CHECK-NEXT: ret double [[TMP1]]
;
entry:
%c = tail call fast double @llvm.maxnum.f64(double %m, double 0.000000e+00)
>From 5111940224f73f598dda4abf289ce70f93a0512b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 21:46:21 +0100
Subject: [PATCH 5/6] [LV/test] Flip to wrap-false in a test to vectorize
---
...aved-accesses-requiring-scev-predicates.ll | 55 +++++++++++++------
1 file changed, 37 insertions(+), 18 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
index a869a177aca21..b8ba2e3ed8851 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
@@ -117,24 +117,43 @@ exit:
ret void
}
-; For %gep, we have the following SCEV: ((4 * (zext i4 {0,+,5}<%loop> to i64))<nuw><nsw> + %x).
-; Note the i4 bit wide AddRec {0,+,5}. It is known to wrap in the loop with trip count 16.
+; For %gep, we an SCEV that is known not to wrap in the loop with the given trip count.
define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr noalias %x, ptr noalias %out) {
; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(
; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 5
-; CHECK-NEXT: [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV_MUL5_MASKED]]
-; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
-; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 16
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[START:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i64> [[TMP0]], splat (i64 9)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 2
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP13]], i32 3
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -145,7 +164,7 @@ loop:
%iv = phi i64 [ 0, %start ], [ %iv.next, %loop ]
%iv.next = add nuw nsw i64 %iv, 1
%iv.mul5 = mul nuw nsw i64 %iv, 5
- %iv.mul5.masked = and i64 %iv.mul5, 15
+ %iv.mul5.masked = and i64 %iv.mul5, 9
%gep = getelementptr inbounds nuw i32, ptr %x, i64 %iv.mul5.masked
%v = load i32, ptr %gep, align 4
%out.i = getelementptr inbounds nuw i32, ptr %out, i64 %iv
@@ -190,7 +209,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -206,7 +225,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 65b573e80cc5685d4195430f8f18918218e1e3d7 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 29 Jun 2026 12:14:24 +0100
Subject: [PATCH 6/6] [LV] Test updates after rebase
---
...predicated-loads-with-predicated-stores.ll | 210 ++++++------------
...aved-accesses-requiring-scev-predicates.ll | 6 +-
2 files changed, 71 insertions(+), 145 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index e11c8d08f7e3f..508f4765b7a37 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -932,71 +932,34 @@ exit:
define void @sink_multiple_store_groups_noalias_via_scev_urem(ptr %dst, ptr %src) {
; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_urem(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
-; CHECK-NEXT: br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE3]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META92:![0-9]+]], !noalias [[META95:![0-9]+]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT: [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
-; CHECK-NEXT: [[TMP15:%.*]] = urem <2 x i64> [[VEC_IND]], splat (i64 128)
-; CHECK-NEXT: [[TMP20:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
-; CHECK-NEXT: [[TMP18:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i64 0
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP19]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <2 x double> [[TMP18]], i64 1
-; CHECK-NEXT: store double [[TMP34]], ptr [[TMP32]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
+; CHECK-NEXT: [[C:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT: [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[CLAMPED:%.*]] = urem i64 [[TMP1]], 128
; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT: store double [[V_1]], ptr [[TMP19]], align 8
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 16
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP25]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP25]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META95]]
+; CHECK-NEXT: [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
+; CHECK-NEXT: [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT: store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
-; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[TMP1]], 16
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -1128,20 +1091,20 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META92:![0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT: store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META99]]
+; CHECK-NEXT: store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META95:![0-9]+]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; CHECK: [[PRED_STORE_IF2]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT: store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT: store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META95]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: [[TMP10:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -1149,14 +1112,14 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT: store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT: store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META95]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; CHECK: [[PRED_STORE_IF6]]:
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT: store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT: store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META95]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
; CHECK-NEXT: [[TMP16:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 9)
@@ -1164,19 +1127,19 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
; CHECK: [[PRED_STORE_IF8]]:
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT: store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT: store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META95]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE9]]
; CHECK: [[PRED_STORE_CONTINUE9]]:
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP16]], i64 1
; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
; CHECK: [[PRED_STORE_IF10]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT: store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT: store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META95]], !noalias [[META92]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; CHECK: [[PRED_STORE_CONTINUE11]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP104:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
@@ -1250,7 +1213,7 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE21:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META106:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
@@ -1258,8 +1221,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109:![0-9]+]], !noalias [[META111:![0-9]+]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META104:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
@@ -1267,8 +1230,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
; CHECK: [[PRED_LOAD_IF10]]:
-; CHECK-NEXT: store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
-; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP11]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]
; CHECK: [[PRED_LOAD_CONTINUE11]]:
@@ -1277,30 +1240,30 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[TMP17]], i64 0
-; CHECK-NEXT: store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
; CHECK: [[PRED_STORE_IF12]]:
; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[TMP17]], i64 1
-; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE13]]
; CHECK: [[PRED_STORE_CONTINUE13]]:
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]
; CHECK: [[PRED_LOAD_IF14]]:
-; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
-; CHECK-NEXT: store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
; CHECK: [[PRED_LOAD_CONTINUE15]]:
; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i32> [ poison, %[[PRED_STORE_CONTINUE13]] ], [ [[TMP24]], %[[PRED_LOAD_IF14]] ]
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_LOAD_IF16:.*]], label %[[PRED_LOAD_CONTINUE17:.*]]
; CHECK: [[PRED_LOAD_IF16]]:
-; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP25]], i64 1
-; CHECK-NEXT: store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT: store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE17]]
; CHECK: [[PRED_LOAD_CONTINUE17]]:
; CHECK-NEXT: [[TMP31:%.*]] = phi <2 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE15]] ], [ [[TMP23]], %[[PRED_LOAD_IF16]] ]
@@ -1309,19 +1272,19 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
; CHECK: [[PRED_STORE_IF18]]:
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x i32> [[TMP32]], i64 0
-; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META113:![0-9]+]], !noalias [[META106]]
+; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META106:![0-9]+]], !noalias [[META99]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE19]]
; CHECK: [[PRED_STORE_CONTINUE19]]:
; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF20:.*]], label %[[PRED_STORE_CONTINUE21]]
; CHECK: [[PRED_STORE_IF20]]:
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x i32> [[TMP32]], i64 1
-; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META113]], !noalias [[META106]]
+; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META106]], !noalias [[META99]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE21]]
; CHECK: [[PRED_STORE_CONTINUE21]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP114:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP107:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
@@ -1389,16 +1352,16 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE13:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META116:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META109:![0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META119:![0-9]+]], !noalias [[META121:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META112:![0-9]+]], !noalias [[META114:![0-9]+]]
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
-; CHECK-NEXT: store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT: store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META112]], !noalias [[META114]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_STORE_IF]] ]
@@ -1406,9 +1369,9 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
; CHECK: [[PRED_STORE_IF10]]:
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP13]], i64 1
-; CHECK-NEXT: store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT: store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; CHECK: [[PRED_STORE_CONTINUE11]]:
; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], %[[PRED_STORE_CONTINUE]] ], [ [[TMP12]], %[[PRED_STORE_IF10]] ]
@@ -1417,8 +1380,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT: store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT: store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP20]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
@@ -1427,8 +1390,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13]]
; CHECK: [[PRED_LOAD_IF12]]:
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT: store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT: store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP24]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE13]]
; CHECK: [[PRED_LOAD_CONTINUE13]]:
@@ -1438,12 +1401,12 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP29:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP26]], <2 x i32> [[TMP15]]
; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i32> [[TMP29]], i64 0
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META123:![0-9]+]], !noalias [[META116]]
+; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META116:![0-9]+]], !noalias [[META109]]
; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i32> [[TMP29]], i64 1
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META123]], !noalias [[META116]]
+; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META116]], !noalias [[META109]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP124:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP117:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
@@ -1492,71 +1455,34 @@ exit:
define void @sink_multiple_store_groups_noalias_via_scev_srem(ptr %dst, ptr %src) {
; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_srem(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
-; CHECK-NEXT: br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE7]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META129:![0-9]+]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT: [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
-; CHECK-NEXT: [[TMP15:%.*]] = srem <2 x i64> [[VEC_IND]], splat (i64 128)
-; CHECK-NEXT: [[TMP18:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
-; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP18]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP28]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
-; CHECK-NEXT: store double [[TMP25]], ptr [[TMP32]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
+; CHECK-NEXT: [[C:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT: [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[CLAMPED:%.*]] = srem i64 [[TMP1]], 128
; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT: store double [[V_1]], ptr [[TMP28]], align 8
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i64 16
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP29]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP29]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
-; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META129]]
+; CHECK-NEXT: [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
+; CHECK-NEXT: [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT: store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
-; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP131:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[TMP1]], 16
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
index b8ba2e3ed8851..2631ab3f0db2f 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
@@ -131,12 +131,12 @@ define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr
; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i64> [[TMP0]], splat (i64 9)
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
More information about the llvm-commits
mailing list