[llvm] [AArch64][SVE] Use loop.dependence.war.mask in vector.memcheck (PR #175943)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 10 08:37:48 PST 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/175943
>From 1ffc45ce7d16149926c98d02eca3e8a52d1691a3 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 12 Jan 2026 14:55:44 +0000
Subject: [PATCH 1/4] [AArch64][SVE] Use `loop.dependence.war.mask` in
vector.memcheck
This patch updates `addDiffRuntimeChecks()` to use the last element
of a `loop.dependence.war.mask` for the check when it is "cheap". This
is currently determined by the cost model returning a cost <= 1.
Using the last element of a `loop.dependence.war.mask` has semantics
similar to the current diff checks, allows for the legal `Diff == 0`
case.
The current checks are equivalent to:
```
if (Diff < 0 || Diff >= AccessSize * (VF * IC))
// vector loop
```
Whereas using `loop.dependence.war.mask` is equivalent to:
```
if (Diff <= 0 || Diff > AccessSize * (VF * IC - 1))
// vector loop
```
Note: `Diff >= AccessSize * (VF * IC)` is the same as
`Diff > AccessSize * (VF * IC - 1)` assuming aligned pointers.
On AArch64 with SVE2 this allows the diff checks to be lowered to:
```
whilewr p0.s, x1, x2
b.nlast .Lscalar_loop
```
Note: This is similar to https://reviews.llvm.org/D138652
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 3 +
.../include/llvm/Transforms/Utils/LoopUtils.h | 9 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 85 ++++--
.../Transforms/Vectorize/LoopVectorize.cpp | 12 +-
.../LoopVectorize/AArch64/alias_mask.ll | 284 ++++++++++++++++++
.../LoopVectorize/AArch64/sve2-histcnt.ll | 27 +-
6 files changed, 375 insertions(+), 45 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 6d27cabf404f8..cfda29888798e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -905,6 +905,9 @@ class TargetTransformInfoImplBase {
switch (ICA.getID()) {
default:
break;
+ case Intrinsic::loop_dependence_raw_mask:
+ case Intrinsic::loop_dependence_war_mask:
+ return 10;
case Intrinsic::allow_runtime_check:
case Intrinsic::allow_ubsan_check:
case Intrinsic::annotation:
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 1e48eeca72952..5546331143688 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -632,9 +632,12 @@ addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
SCEVExpander &Expander, bool HoistRuntimeChecks = false);
-LLVM_ABI Value *addDiffRuntimeChecks(
- Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
- function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC);
+LLVM_ABI Value *addDiffRuntimeChecks(Instruction *Loc,
+ ArrayRef<PointerDiffInfo> Checks,
+ SCEVExpander &Expander, ElementCount VF,
+ unsigned IC,
+ const TargetTransformInfo &TTI,
+ TTI::TargetCostKind CostKind);
/// Struct to hold information about a partially invariant condition.
struct IVConditionInfo {
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 437465b30e1c2..a5b0a3fd06e1f 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2173,38 +2173,85 @@ Value *llvm::addRuntimeChecks(
return MemoryRuntimeCheck;
}
-Value *llvm::addDiffRuntimeChecks(
- Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
- function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC) {
+Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
+ ArrayRef<PointerDiffInfo> Checks,
+ SCEVExpander &Expander, ElementCount VF,
+ unsigned IC, const TargetTransformInfo &TTI,
+ TTI::TargetCostKind CostKind) {
LLVMContext &Ctx = Loc->getContext();
IRBuilder ChkBuilder(Ctx, InstSimplifyFolder(Loc->getDataLayout()));
ChkBuilder.SetInsertPoint(Loc);
+ Value *RuntimeVF = nullptr;
// Our instructions might fold to a constant.
Value *MemoryRuntimeCheck = nullptr;
-
auto &SE = *Expander.getSE();
+
+ constexpr Intrinsic::ID LoopDeskMaskIID = Intrinsic::loop_dependence_war_mask;
+ auto LoopDepMaskIsCheap = [&](Type *MaskTy, Value *AccessSize) {
+ Value *NullPtr = ConstantPointerNull::get(PointerType::getUnqual(Ctx));
+ // The pointer values should not change the cost. The access size (constant)
+ // is needed to by targets to cost the mask.
+ IntrinsicCostAttributes ICA(LoopDeskMaskIID, MaskTy,
+ {NullPtr, NullPtr, AccessSize});
+ InstructionCost Cost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+ return Cost.isValid() && Cost <= 1;
+ };
+
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
+ Value *IsConflict;
+ Module *M = Loc->getModule();
Type *Ty = SinkStart->getType();
- // Compute VF * IC * AccessSize.
- auto *VFTimesICTimesSize =
- ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, IC * AccessSize));
- Value *Diff =
- Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
-
- // Check if the same compare has already been created earlier. In that case,
- // there is no need to check it again.
- Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
- if (IsConflict)
- continue;
+ Type *CheckTy = ChkBuilder.getIntNTy(Ty->getScalarSizeInBits());
+
+ VectorType *MaskTy = VectorType::get(ChkBuilder.getInt1Ty(), VF * IC);
+ Value *LoopAccessSize = ChkBuilder.getInt64(AccessSize);
+ if (!LoopDepMaskIsCheap(MaskTy, LoopAccessSize)) {
+ // Compute VF * IC * AccessSize.
+ if (!RuntimeVF)
+ RuntimeVF = ChkBuilder.CreateElementCount(CheckTy, VF);
+ auto *VFTimesICTimesSize = ChkBuilder.CreateMul(
+ RuntimeVF, ConstantInt::get(Ty, IC * AccessSize));
+ Value *Diff =
+ Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
+
+ // Check if the same compare has already been created earlier. In that
+ // case, there is no need to check it again.
+ IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
+ if (IsConflict)
+ continue;
- IsConflict =
- ChkBuilder.CreateICmpULT(Diff, VFTimesICTimesSize, "diff.check");
- SeenCompares.insert({{Diff, VFTimesICTimesSize}, IsConflict});
+ IsConflict =
+ ChkBuilder.CreateICmpULT(Diff, VFTimesICTimesSize, "diff.check");
+ SeenCompares.insert({{Diff, VFTimesICTimesSize}, IsConflict});
+ } else {
+ Function *LoopDepMaskIntr =
+ Intrinsic::getOrInsertDeclaration(M, LoopDeskMaskIID, {MaskTy});
+
+ Value *Src = Expander.expandCodeFor(SrcStart, Ty, Loc);
+ Value *SrcPtr = ChkBuilder.CreateIntToPtr(Src, ChkBuilder.getPtrTy());
+ Value *Sink = Expander.expandCodeFor(SinkStart, Ty, Loc);
+ Value *SinkPtr = ChkBuilder.CreateIntToPtr(Sink, ChkBuilder.getPtrTy());
+ Value *Mask = ChkBuilder.CreateCall(
+ LoopDepMaskIntr->getFunctionType(), LoopDepMaskIntr,
+ {SrcPtr, SinkPtr, LoopAccessSize}, "loop.dep.mask");
+
+ IsConflict = SeenCompares.lookup({Mask, nullptr});
+ if (IsConflict)
+ continue;
+
+ Value *LastLaneIdx = ChkBuilder.CreateSub(
+ ChkBuilder.CreateElementCount(CheckTy, MaskTy->getElementCount()),
+ ChkBuilder.getIntN(Ty->getScalarSizeInBits(), 1));
+ Value *NoConflict =
+ ChkBuilder.CreateExtractElement(Mask, LastLaneIdx, "no.conflict");
+
+ IsConflict = ChkBuilder.CreateNot(NoConflict, "is.conflict");
+ SeenCompares.insert({{Mask, nullptr}, IsConflict});
+ }
if (NeedsFreeze)
IsConflict =
ChkBuilder.CreateFreeze(IsConflict, IsConflict->getName() + ".fr");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 46882ebf7f4a7..3507899a4399b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1883,15 +1883,9 @@ class GeneratedRTChecks {
auto DiffChecks = RtPtrChecking.getDiffChecks();
if (DiffChecks) {
- Value *RuntimeVF = nullptr;
- MemRuntimeCheckCond = addDiffRuntimeChecks(
- MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp,
- [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
- if (!RuntimeVF)
- RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
- return RuntimeVF;
- },
- IC);
+ MemRuntimeCheckCond =
+ addDiffRuntimeChecks(MemCheckBlock->getTerminator(), *DiffChecks,
+ MemCheckExp, VF, IC, *TTI, CostKind);
} else {
MemRuntimeCheckCond = addRuntimeChecks(
MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
new file mode 100644
index 0000000000000..2c16dded9dff8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
@@ -0,0 +1,284 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --filter-out-after "^scalar.ph:" --version 4
+; RUN: opt -S -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=1 %s | FileCheck %s
+
+define dso_local void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define dso_local void @alias_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[B]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP19:%.*]] = mul nuw i64 [[TMP5]], 16
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP19]], 1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 16 x i1> [[LOOP_DEP_MASK]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP24]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[N]], [[TMP14]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp ugt i64 [[N]], [[TMP14]]
+; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP17]], i64 [[TMP15]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP16]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP20]], ptr align 1 [[TMP21]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP13]])
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT: [[TMP26:%.*]] = xor i1 [[TMP23]], true
+; CHECK-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %add = add i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %for.body, %entry
+ ret void
+}
+
+define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define i32 @alias_mask_read_after_write(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[C]], ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP20:%.*]] = mul nuw i64 [[TMP13]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP20]], 1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: br i1 [[TMP10]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP9]], 2
+; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP27]], 2
+; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[N]], [[TMP18]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ugt i64 [[N]], [[TMP18]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP16]], i64 [[TMP15]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP17]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 2 [[TMP19]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP21]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP23]], [[WIDE_MASKED_LOAD5]]
+; CHECK-NEXT: [[TMP25]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP24]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP14]])
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT: [[TMP26:%.*]] = xor i1 [[TMP28]], true
+; CHECK-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+;
+entry:
+ %cmp19 = icmp sgt i64 %n, 0
+ br i1 %cmp19, label %for.body, label %exit
+
+for.body: ; preds = %entry, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load.a = load i32, ptr %gep.a, align 2
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %load.a, ptr %gep.c, align 2
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %load.b = load i32, ptr %gep.b, align 2
+ %add = add i32 %load.a, %accum
+ %add2 = add i32 %add, %load.b
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %entry, %for.body
+ %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ ret i32 %result
+}
+
+define dso_local void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define dso_local void @alias_mask_multiple(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[A]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP32:%.*]] = mul nuw i64 [[TMP31]], 16
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP32]], 1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 16 x i1> [[LOOP_DEP_MASK]], i64 [[TMP2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[B]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP6]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[NO_CONFLICT5:%.*]] = extractelement <vscale x 16 x i1> [[LOOP_DEP_MASK4]], i64 [[TMP5]]
+; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = xor i1 [[NO_CONFLICT5]], true
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP19]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP18]], 4
+; CHECK-NEXT: [[TMP21:%.*]] = sub i64 [[N]], [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = icmp ugt i64 [[N]], [[TMP20]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i64 [[TMP21]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP25]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP26]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP27:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD14]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP27]], ptr align 1 [[TMP28]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP23]])
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
+; CHECK-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %add = add i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %for.body, %entry
+ ret void
+}
+
+define i32 @alias_mask_multiple_read_after_write(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define i32 @alias_mask_multiple_read_after_write(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[A]], ptr [[C]], i64 4)
+; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP34]], 1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[C]], ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP6]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[NO_CONFLICT5:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK4]], i64 [[TMP5]]
+; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = xor i1 [[NO_CONFLICT5]], true
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP19]], 2
+; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP24]], 2
+; CHECK-NEXT: [[TMP21:%.*]] = sub i64 [[N]], [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = icmp ugt i64 [[N]], [[TMP20]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i64 [[TMP21]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP25]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 2 [[TMP26]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP29:%.*]] = add <vscale x 4 x i32> [[TMP28]], [[WIDE_MASKED_LOAD14]]
+; CHECK-NEXT: [[TMP30]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP29]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP23]])
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT: [[TMP32:%.*]] = xor i1 [[TMP31]], true
+; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+;
+entry:
+ %cmp19 = icmp sgt i64 %n, 0
+ br i1 %cmp19, label %for.body, label %exit
+
+for.body: ; preds = %entry, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load.a = load i32, ptr %gep.a, align 2
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %load.a, ptr %gep.c, align 2
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %load.b = load i32, ptr %gep.b, align 2
+ %add = add i32 %load.a, %accum
+ %add2 = add i32 %add, %load.b
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %entry, %for.body
+ %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ ret i32 %result
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 1193e118bf9a5..a936c43982b00 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -307,7 +307,7 @@ define void @histogram_float(ptr noalias %buckets, ptr readonly %indices, i64 %N
; CHECK-NEXT: store float [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -350,7 +350,7 @@ define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %ind
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP12]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP14]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -406,7 +406,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP21]], i32 1, <vscale x 4 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -459,7 +459,7 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP11]], i32 1, <vscale x 4 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -513,7 +513,7 @@ define void @histogram_array_4op_gep_nonzero_const_idx(i64 noundef %N, ptr reado
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP7]], i32 1, <vscale x 4 x i1> splat (i1 true))
; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -564,7 +564,7 @@ define void @simple_histogram_tailfold(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_EXIT:%.*]]
; CHECK: for.exit:
@@ -602,13 +602,12 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[ARRAY1:%.*]] = ptrtoint ptr [[ARRAY]] to i64
-; CHECK-NEXT: [[INDICES2:%.*]] = ptrtoint ptr [[INDICES]] to i64
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[INDICES]], ptr [[ARRAY]], i64 4)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[ARRAY1]], [[INDICES2]]
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
-; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP5]], -1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP4]]
+; CHECK-NEXT: br i1 [[NO_CONFLICT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
@@ -632,7 +631,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -725,7 +724,7 @@ define void @simple_histogram_64b(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> [[TMP6]], i64 1, <vscale x 2 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
>From dda9d40b785b262a1025c80d898f57c96b6c556e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 9 Feb 2026 18:35:09 +0000
Subject: [PATCH 2/4] WIP
---
llvm/include/llvm/Analysis/LoopAccessAnalysis.h | 5 +++--
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 7 ++++++-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 3 ++-
3 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index c85ef3e131068..92ea3b53d59b4 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -491,11 +491,12 @@ struct PointerDiffInfo {
const SCEV *SinkStart;
unsigned AccessSize;
bool NeedsFreeze;
+ Align AccessAlign;
PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart,
- unsigned AccessSize, bool NeedsFreeze)
+ unsigned AccessSize, bool NeedsFreeze, Align AccessAlign)
: SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize),
- NeedsFreeze(NeedsFreeze) {}
+ NeedsFreeze(NeedsFreeze), AccessAlign(AccessAlign) {}
};
/// Holds information about the memory runtime legality checks to verify
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 2fc724970747c..1164a982dc2ff 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -512,11 +512,16 @@ bool RuntimePointerChecking::tryToCreateDiffCheck(
}
}
+ // Find the minimum common alignment of all accesses.
+ Align AccessAlign = getLoadStoreAlignment(SrcInsts[0]);
+ for (Instruction *Inst : concat<Instruction *>(SrcInsts, SinkInsts))
+ AccessAlign = std::min(AccessAlign, getLoadStoreAlignment(Inst));
+
LLVM_DEBUG(dbgs() << "LAA: Creating diff runtime check for:\n"
<< "SrcStart: " << *SrcStartInt << '\n'
<< "SinkStartInt: " << *SinkStartInt << '\n');
DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
- Src->NeedsFreeze || Sink->NeedsFreeze);
+ Src->NeedsFreeze || Sink->NeedsFreeze, AccessAlign);
return true;
}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index a5b0a3fd06e1f..991cec386f7b3 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2201,7 +2201,8 @@ Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
- for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
+ for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze, AccessAlign] :
+ Checks) {
Value *IsConflict;
Module *M = Loc->getModule();
Type *Ty = SinkStart->getType();
>From 2ad062e5741acc7f49e0fb13ae7b60fc0cac21f7 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 10 Feb 2026 16:29:34 +0000
Subject: [PATCH 3/4] Fixups
---
.../include/llvm/Transforms/Utils/LoopUtils.h | 10 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 57 ++---
.../Transforms/Vectorize/LoopVectorize.cpp | 21 +-
.../CostModel/loop-dep-mask-no_info.ll | 47 ++++
.../LoopVectorize/AArch64/alias_mask.ll | 175 ++++++++-----
.../AArch64/conditional-branches-cost.ll | 4 +-
.../AArch64/induction-costs-sve.ll | 4 +-
.../AArch64/scalable-struct-return.ll | 6 +-
.../LoopVectorize/AArch64/sve-fneg.ll | 2 +-
.../LoopVectorize/AArch64/sve-multiexit.ll | 4 +-
.../sve-runtime-check-size-based-threshold.ll | 10 +-
.../AArch64/sve-vector-reverse.ll | 2 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 45 +++-
...ntime-checks-difference-simplifications.ll | 233 ++++++++++++++++++
.../scalable-first-order-recurrence.ll | 8 +-
15 files changed, 508 insertions(+), 120 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/loop-dep-mask-no_info.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 5546331143688..2f3317e533e12 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -632,12 +632,10 @@ addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
SCEVExpander &Expander, bool HoistRuntimeChecks = false);
-LLVM_ABI Value *addDiffRuntimeChecks(Instruction *Loc,
- ArrayRef<PointerDiffInfo> Checks,
- SCEVExpander &Expander, ElementCount VF,
- unsigned IC,
- const TargetTransformInfo &TTI,
- TTI::TargetCostKind CostKind);
+LLVM_ABI Value *addDiffRuntimeChecks(
+ Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
+ ElementCount VF, unsigned IC,
+ function_ref<bool(unsigned)> UsesLoopDependenceMaskForAccessSize);
/// Struct to hold information about a partially invariant condition.
struct IVConditionInfo {
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 991cec386f7b3..587172b901cd7 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2173,12 +2173,10 @@ Value *llvm::addRuntimeChecks(
return MemoryRuntimeCheck;
}
-Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
- ArrayRef<PointerDiffInfo> Checks,
- SCEVExpander &Expander, ElementCount VF,
- unsigned IC, const TargetTransformInfo &TTI,
- TTI::TargetCostKind CostKind) {
-
+Value *llvm::addDiffRuntimeChecks(
+ Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
+ ElementCount VF, unsigned IC,
+ function_ref<bool(unsigned)> UsesLoopDependenceMaskForAccessSize) {
LLVMContext &Ctx = Loc->getContext();
IRBuilder ChkBuilder(Ctx, InstSimplifyFolder(Loc->getDataLayout()));
ChkBuilder.SetInsertPoint(Loc);
@@ -2187,38 +2185,26 @@ Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
Value *MemoryRuntimeCheck = nullptr;
auto &SE = *Expander.getSE();
- constexpr Intrinsic::ID LoopDeskMaskIID = Intrinsic::loop_dependence_war_mask;
- auto LoopDepMaskIsCheap = [&](Type *MaskTy, Value *AccessSize) {
- Value *NullPtr = ConstantPointerNull::get(PointerType::getUnqual(Ctx));
- // The pointer values should not change the cost. The access size (constant)
- // is needed to by targets to cost the mask.
- IntrinsicCostAttributes ICA(LoopDeskMaskIID, MaskTy,
- {NullPtr, NullPtr, AccessSize});
- InstructionCost Cost = TTI.getIntrinsicInstrCost(ICA, CostKind);
- return Cost.isValid() && Cost <= 1;
- };
-
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze, AccessAlign] :
Checks) {
Value *IsConflict;
- Module *M = Loc->getModule();
Type *Ty = SinkStart->getType();
Type *CheckTy = ChkBuilder.getIntNTy(Ty->getScalarSizeInBits());
+ Value *Diff =
+ Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
VectorType *MaskTy = VectorType::get(ChkBuilder.getInt1Ty(), VF * IC);
- Value *LoopAccessSize = ChkBuilder.getInt64(AccessSize);
- if (!LoopDepMaskIsCheap(MaskTy, LoopAccessSize)) {
+ if (!UsesLoopDependenceMaskForAccessSize(AccessSize) ||
+ commonAlignment(AccessAlign, AccessSize) < AccessSize) {
// Compute VF * IC * AccessSize.
if (!RuntimeVF)
RuntimeVF = ChkBuilder.CreateElementCount(CheckTy, VF);
+
auto *VFTimesICTimesSize = ChkBuilder.CreateMul(
RuntimeVF, ConstantInt::get(Ty, IC * AccessSize));
- Value *Diff =
- Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
-
// Check if the same compare has already been created earlier. In that
// case, there is no need to check it again.
IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
@@ -2229,21 +2215,20 @@ Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
ChkBuilder.CreateICmpULT(Diff, VFTimesICTimesSize, "diff.check");
SeenCompares.insert({{Diff, VFTimesICTimesSize}, IsConflict});
} else {
- Function *LoopDepMaskIntr =
- Intrinsic::getOrInsertDeclaration(M, LoopDeskMaskIID, {MaskTy});
-
- Value *Src = Expander.expandCodeFor(SrcStart, Ty, Loc);
- Value *SrcPtr = ChkBuilder.CreateIntToPtr(Src, ChkBuilder.getPtrTy());
- Value *Sink = Expander.expandCodeFor(SinkStart, Ty, Loc);
- Value *SinkPtr = ChkBuilder.CreateIntToPtr(Sink, ChkBuilder.getPtrTy());
- Value *Mask = ChkBuilder.CreateCall(
- LoopDepMaskIntr->getFunctionType(), LoopDepMaskIntr,
- {SrcPtr, SinkPtr, LoopAccessSize}, "loop.dep.mask");
-
- IsConflict = SeenCompares.lookup({Mask, nullptr});
+ Value *LoopAccessSize = ChkBuilder.getInt64(AccessSize);
+ IsConflict = SeenCompares.lookup({Diff, LoopAccessSize});
if (IsConflict)
continue;
+ // Note: This creates loop.dependence.war.mask(ptr null, ptr %diff). This
+ // allows SCEV to remove common offsets and avoids creating duplicate
+ // checks. If %diff is a sub, it can be folded into the mask.
+ Value *SrcPtr = ConstantPointerNull::get(PointerType::getUnqual(Ctx));
+ Value *SinkPtr = ChkBuilder.CreateIntToPtr(Diff, ChkBuilder.getPtrTy());
+ Value *Mask = ChkBuilder.CreateIntrinsic(
+ MaskTy, Intrinsic::loop_dependence_war_mask,
+ {SrcPtr, SinkPtr, LoopAccessSize}, {}, "loop.dep.mask");
+
Value *LastLaneIdx = ChkBuilder.CreateSub(
ChkBuilder.CreateElementCount(CheckTy, MaskTy->getElementCount()),
ChkBuilder.getIntN(Ty->getScalarSizeInBits(), 1));
@@ -2251,7 +2236,7 @@ Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
ChkBuilder.CreateExtractElement(Mask, LastLaneIdx, "no.conflict");
IsConflict = ChkBuilder.CreateNot(NoConflict, "is.conflict");
- SeenCompares.insert({{Mask, nullptr}, IsConflict});
+ SeenCompares.insert({{Diff, LoopAccessSize}, IsConflict});
}
if (NeedsFreeze)
IsConflict =
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3507899a4399b..8ba5c32dbc7b7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1883,9 +1883,13 @@ class GeneratedRTChecks {
auto DiffChecks = RtPtrChecking.getDiffChecks();
if (DiffChecks) {
+ LLVMContext &Ctx = MemCheckBlock->getContext();
+ auto UseLoopDependenceMask = [&](unsigned AccessSize) {
+ return isLoopDependenceMaskCheap(Ctx, VF, IC, AccessSize);
+ };
MemRuntimeCheckCond =
addDiffRuntimeChecks(MemCheckBlock->getTerminator(), *DiffChecks,
- MemCheckExp, VF, IC, *TTI, CostKind);
+ MemCheckExp, VF, IC, UseLoopDependenceMask);
} else {
MemRuntimeCheckCond = addRuntimeChecks(
MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
@@ -1937,6 +1941,21 @@ class GeneratedRTChecks {
OuterLoop = L->getParentLoop();
}
+ bool isLoopDependenceMaskCheap(LLVMContext &Ctx, ElementCount VF, unsigned IC,
+ unsigned AccessSize) {
+ if (ForceTargetInstructionCost.getNumOccurrences() > 0)
+ return ForceTargetInstructionCost <= 1;
+ VectorType *MaskTy = VectorType::get(Type::getInt1Ty(Ctx), VF * IC);
+ Value *AccessSizeVal = ConstantInt::get(Type::getInt64Ty(Ctx), AccessSize);
+ Value *NullPtr = ConstantPointerNull::get(PointerType::getUnqual(Ctx));
+ // The pointer values should not change the cost. The access size (constant)
+ // is needed to by targets to cost the mask.
+ IntrinsicCostAttributes ICA(Intrinsic::loop_dependence_war_mask, MaskTy,
+ {NullPtr, NullPtr, AccessSizeVal});
+ InstructionCost Cost = TTI->getIntrinsicInstrCost(ICA, CostKind);
+ return Cost.isValid() && Cost <= 1;
+ }
+
InstructionCost getCost() {
if (SCEVCheckBlock || MemCheckBlock)
LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
diff --git a/llvm/test/Analysis/CostModel/loop-dep-mask-no_info.ll b/llvm/test/Analysis/CostModel/loop-dep-mask-no_info.ll
new file mode 100644
index 0000000000000..7b326c7b4a805
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/loop-dep-mask-no_info.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size %s -S -o - | FileCheck %s --check-prefix=CHECK-SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput %s -S -o - | FileCheck %s --check-prefix=CHECK-THROUGHPUT
+
+define void @loop_dependence_war_mask(ptr %a, ptr %b) {
+; CHECK-SIZE-LABEL: 'loop_dependence_war_mask'
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 1)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 2)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 4)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 8)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; CHECK-THROUGHPUT-LABEL: 'loop_dependence_war_mask'
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 1)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 2)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 4)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 8)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %res1 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 1)
+ %res2 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 2)
+ %res3 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 4)
+ %res4 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 8)
+ ret void
+}
+
+define void @loop_dependence_raw_mask(ptr %a, ptr %b) {
+; CHECK-SIZE-LABEL: 'loop_dependence_raw_mask'
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1(ptr %a, ptr %b, i64 1)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1(ptr %a, ptr %b, i64 2)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1(ptr %a, ptr %b, i64 4)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1(ptr %a, ptr %b, i64 8)
+; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; CHECK-THROUGHPUT-LABEL: 'loop_dependence_raw_mask'
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1(ptr %a, ptr %b, i64 1)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1(ptr %a, ptr %b, i64 2)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1(ptr %a, ptr %b, i64 4)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1(ptr %a, ptr %b, i64 8)
+; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %res1 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1(ptr %a, ptr %b, i64 1)
+ %res2 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1(ptr %a, ptr %b, i64 2)
+ %res3 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1(ptr %a, ptr %b, i64 4)
+ %res4 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1(ptr %a, ptr %b, i64 8)
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
index 2c16dded9dff8..e4ea5b2c7d0db 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias_mask.ll
@@ -1,16 +1,17 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --filter-out-after "^scalar.ph:" --version 4
; RUN: opt -S -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=1 %s | FileCheck %s
-define dso_local void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
-; CHECK-LABEL: define dso_local void @alias_mask(
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[B]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[C1]], [[B2]]
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr null, ptr [[TMP1]], i64 1)
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP19:%.*]] = mul nuw i64 [[TMP5]], 16
; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP19]], 1
@@ -30,13 +31,13 @@ define dso_local void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP16]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP20]], ptr align 1 [[TMP21]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP12]], ptr align 1 [[TMP16]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP13]])
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
@@ -45,8 +46,7 @@ define dso_local void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: middle.block:
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -69,17 +69,15 @@ define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n)
; CHECK-LABEL: define i32 @alias_mask_read_after_write(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-NEXT: [[C2:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT: [[B1:%.*]] = ptrtoint ptr [[B]] to i64
; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[C]], ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[C2]]
; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP20:%.*]] = mul nuw i64 [[TMP13]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP20]], 1
-; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP10]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
@@ -112,8 +110,7 @@ define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n)
; CHECK: middle.block:
;
entry:
- %cmp19 = icmp sgt i64 %n, 0
- br i1 %cmp19, label %for.body, label %exit
+ br label %for.body
for.body: ; preds = %entry, %for.body
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -131,26 +128,29 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %exit, label %for.body
exit: ; preds = %entry, %for.body
- %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
- ret i32 %result
+ ret i32 %add2
}
-define dso_local void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
-; CHECK-LABEL: define dso_local void @alias_mask_multiple(
+define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask_multiple(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-NEXT: [[B3:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[A]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[C1]], [[A2]]
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr null, ptr [[TMP1]], i64 1)
; CHECK-NEXT: [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP32:%.*]] = mul nuw i64 [[TMP31]], 16
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP32]], 1
; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 16 x i1> [[LOOP_DEP_MASK]], i64 [[TMP2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = xor i1 [[NO_CONFLICT]], true
-; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[B]], ptr [[C]], i64 1)
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[C1]], [[B3]]
+; CHECK-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr null, ptr [[TMP8]], i64 1)
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP6]], 16
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
@@ -186,8 +186,7 @@ define dso_local void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK: middle.block:
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -210,21 +209,25 @@ define i32 @alias_mask_multiple_read_after_write(ptr %a, ptr %b, ptr %c, i64 %n)
; CHECK-LABEL: define i32 @alias_mask_multiple_read_after_write(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-NEXT: [[B3:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[A]], ptr [[C]], i64 4)
-; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP34]], 1
-; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP2]]
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = xor i1 [[NO_CONFLICT]], true
-; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[C]], ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[C1]], [[A2]]
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr null, ptr [[TMP1]], i64 4)
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[NO_CONFLICT:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK]], i64 [[TMP2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[B3]], [[C1]]
+; CHECK-NEXT: [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK4:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr null, ptr [[TMP10]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP8]], 1
; CHECK-NEXT: [[NO_CONFLICT5:%.*]] = extractelement <vscale x 4 x i1> [[LOOP_DEP_MASK4]], i64 [[TMP5]]
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = xor i1 [[NO_CONFLICT5]], true
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
@@ -244,11 +247,11 @@ define i32 @alias_mask_multiple_read_after_write(ptr %a, ptr %b, ptr %c, i64 %n)
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP25]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP25]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 2 [[TMP26]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP26]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP29:%.*]] = add <vscale x 4 x i32> [[TMP28]], [[WIDE_MASKED_LOAD14]]
; CHECK-NEXT: [[TMP30]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP29]], <vscale x 4 x i32> [[VEC_PHI]]
@@ -260,18 +263,17 @@ define i32 @alias_mask_multiple_read_after_write(ptr %a, ptr %b, ptr %c, i64 %n)
; CHECK: middle.block:
;
entry:
- %cmp19 = icmp sgt i64 %n, 0
- br i1 %cmp19, label %for.body, label %exit
+ br label %for.body
for.body: ; preds = %entry, %for.body
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%accum = phi i32 [ 0, %entry ], [ %add2, %for.body ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %load.a = load i32, ptr %gep.a, align 2
+ %load.a = load i32, ptr %gep.a, align 4
%gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
- store i32 %load.a, ptr %gep.c, align 2
+ store i32 %load.a, ptr %gep.c, align 4
%gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %load.b = load i32, ptr %gep.b, align 2
+ %load.b = load i32, ptr %gep.b, align 4
%add = add i32 %load.a, %accum
%add2 = add i32 %add, %load.b
%iv.next = add nuw nsw i64 %iv, 1
@@ -279,6 +281,67 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %exit, label %for.body
exit: ; preds = %entry, %for.body
- %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
- ret i32 %result
+ ret i32 %add2
+}
+
+; Negative test: We can't use `loop.dependence.war.mask` with unaligned pointers.
+define void @misaligned_alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @misaligned_alias_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[B2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[N]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[N]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP13]], ptr align 2 [[TMP14]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP15]], true
+; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: middle.block:
+;
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load.a = load i32, ptr %gep.a, align 2
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %load.b = load i32, ptr %gep.b, align 2
+ %add = add i32 %load.b, %load.a
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %add, ptr %gep.c, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %for.body, %entry
+ ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 4f441e6834fc7..38088d5224707 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1263,9 +1263,9 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
+; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
; DEFAULT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: [[TMP7:%.*]] = sub i64 [[C1]], [[B3]]
; DEFAULT-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
@@ -1317,9 +1317,9 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
+; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
-; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: [[TMP4:%.*]] = sub i64 [[C1]], [[B3]]
; PRED-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP4]], [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index aa6e4df26d71b..0a9c0c4d2b9d3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -17,10 +17,10 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
+; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
; DEFAULT-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: br i1 [[DIFF_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
@@ -123,9 +123,9 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
+; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
-; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
index 695ea713ed125..280eae0a14d43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
@@ -132,17 +132,17 @@ define void @struct_return_f32_widen_rt_checks(ptr %in, ptr writeonly %out_a, pt
; CHECK-NEXT: [[OUT_B1:%.*]] = ptrtoint ptr [[OUT_B]] to i64
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[OUT_B1]], [[OUT_A2]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[OUT_B1]], [[OUT_A2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 4
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[OUT_A2]], [[IN3]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 4
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP1]], 4
; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[OUT_B1]], [[IN3]]
+; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP1]], 4
; CHECK-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
; CHECK-NEXT: [[CONFLICT_RDX6:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK5]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
index a5f5cbad0d949..e21c113d074aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
@@ -20,10 +20,10 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[D1]], [[S2]]
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[D1]], [[S2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index ad85147a03b42..b87855b7b4b9e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -20,10 +20,10 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
@@ -88,10 +88,10 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 9ff193b36c008..ca1bd3c7b4a84 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -19,25 +19,25 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[DST_21]], [[DST_12]]
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16
-; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[DST_21]], [[DST_12]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[DST_12]], [[SRC_13]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[DST_12]], [[SRC_25]]
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[DIFF_CHECK6:%.*]] = icmp ult i64 [[TMP10]], [[TMP9]]
; CHECK-NEXT: [[CONFLICT_RDX7:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK6]]
-; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[DST_21]], [[SRC_13]]
+; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[DIFF_CHECK8:%.*]] = icmp ult i64 [[TMP12]], [[TMP11]]
; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX7]], [[DIFF_CHECK8]]
-; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[DST_21]], [[SRC_25]]
+; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP4]], 16
; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP14]], [[TMP13]]
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX9]], [[DIFF_CHECK10]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index ba6bd8f36f5e4..9eaeaa432afa2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -90,9 +90,9 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 7
-; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP3]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index a936c43982b00..99ead5d68034e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -269,6 +269,45 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N_VEC]], 0
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP9]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK: vec.epilog.ph:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 3
+; CHECK-NEXT: [[DOTNOT1:%.*]] = sub nsw i64 0, [[TMP12]]
+; CHECK-NEXT: [[N_VEC3:%.*]] = and i64 [[N]], [[DOTNOT1]]
+; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK: vec.epilog.vector.body:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[TMP15]], i8 1, <vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP12]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: vec.epilog.middle.block:
+; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
+; CHECK: vec.epilog.scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i8, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i8 [[L_BUCKET]], 1
+; CHECK-NEXT: store i8 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
;
entry:
br label %for.body
@@ -602,7 +641,11 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[INDICES]], ptr [[ARRAY]], i64 4)
+; CHECK-NEXT: [[ARRAY1:%.*]] = ptrtoint ptr [[ARRAY]] to i64
+; CHECK-NEXT: [[INDICES2:%.*]] = ptrtoint ptr [[INDICES]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[ARRAY1]], [[INDICES2]]
+; CHECK-NEXT: [[TMP10:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT: [[LOOP_DEP_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr null, ptr [[TMP10]], i64 4)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP3]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP5]], -1
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
index e6624da7516b1..0f51c245e3e9e 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -o - -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-target-instruction-cost=1 -force-vector-interleave=1 -o - -S %s | FileCheck %s --check-prefix=CHECK-LOOP-DEP
; Test case with a large number of pointer groups to check for memory
; conflicts, but with many redundant checks that can be simplified.
@@ -149,6 +150,184 @@ define void @test_large_number_of_group(ptr %dst, i64 %off, i64 %N) {
; CHECK: exit:
; CHECK-NEXT: ret void
;
+; CHECK-LOOP-DEP-LABEL: @test_large_number_of_group(
+; CHECK-LOOP-DEP-NEXT: entry:
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_2:%.*]] = shl i64 [[OFF:%.*]], 1
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_3:%.*]] = mul i64 [[OFF]], 3
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_4:%.*]] = shl i64 [[OFF]], 2
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_5:%.*]] = mul i64 [[OFF]], 5
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_6:%.*]] = mul i64 [[OFF]], 6
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_7:%.*]] = mul i64 [[OFF]], 7
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_8:%.*]] = shl i64 [[OFF]], 3
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_9:%.*]] = mul i64 [[OFF]], 9
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_10:%.*]] = mul i64 [[OFF]], 10
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_11:%.*]] = mul i64 [[OFF]], 11
+; CHECK-LOOP-DEP-NEXT: [[OFF_MUL_12:%.*]] = mul i64 [[OFF]], 12
+; CHECK-LOOP-DEP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-LOOP-DEP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-LOOP-DEP: vector.memcheck:
+; CHECK-LOOP-DEP-NEXT: [[TMP0:%.*]] = inttoptr i64 [[OFF_MUL_8]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK1:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP0]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT1:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK1]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT1:%.*]] = xor i1 [[NO_CONFLICT1]], true
+; CHECK-LOOP-DEP-NEXT: [[TMP2:%.*]] = shl i64 [[OFF]], 4
+; CHECK-LOOP-DEP-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP3]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX1:%.*]] = or i1 [[IS_CONFLICT1]], [[IS_CONFLICT]]
+; CHECK-LOOP-DEP-NEXT: [[TMP7:%.*]] = mul i64 [[OFF]], 24
+; CHECK-LOOP-DEP-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP4]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT3:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK2]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT4:%.*]] = xor i1 [[NO_CONFLICT3]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[CONFLICT_RDX1]], [[IS_CONFLICT4]]
+; CHECK-LOOP-DEP-NEXT: [[TMP12:%.*]] = shl i64 [[OFF]], 5
+; CHECK-LOOP-DEP-NEXT: [[TMP6:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK5:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP6]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT6:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK5]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT7:%.*]] = xor i1 [[NO_CONFLICT6]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX8:%.*]] = or i1 [[CONFLICT_RDX]], [[IS_CONFLICT7]]
+; CHECK-LOOP-DEP-NEXT: [[TMP17:%.*]] = mul i64 [[OFF]], 40
+; CHECK-LOOP-DEP-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK9:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP8]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT10:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK9]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT11:%.*]] = xor i1 [[NO_CONFLICT10]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX12:%.*]] = or i1 [[CONFLICT_RDX8]], [[IS_CONFLICT11]]
+; CHECK-LOOP-DEP-NEXT: [[TMP22:%.*]] = mul i64 [[OFF]], 48
+; CHECK-LOOP-DEP-NEXT: [[TMP10:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK13:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP10]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT14:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK13]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT15:%.*]] = xor i1 [[NO_CONFLICT14]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX16:%.*]] = or i1 [[CONFLICT_RDX12]], [[IS_CONFLICT15]]
+; CHECK-LOOP-DEP-NEXT: [[TMP27:%.*]] = mul i64 [[OFF]], 56
+; CHECK-LOOP-DEP-NEXT: [[TMP13:%.*]] = inttoptr i64 [[TMP27]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK17:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP13]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT18:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK17]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT19:%.*]] = xor i1 [[NO_CONFLICT18]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX20:%.*]] = or i1 [[CONFLICT_RDX16]], [[IS_CONFLICT19]]
+; CHECK-LOOP-DEP-NEXT: [[TMP32:%.*]] = shl i64 [[OFF]], 6
+; CHECK-LOOP-DEP-NEXT: [[TMP14:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK21:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP14]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT22:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK21]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT23:%.*]] = xor i1 [[NO_CONFLICT22]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX24:%.*]] = or i1 [[CONFLICT_RDX20]], [[IS_CONFLICT23]]
+; CHECK-LOOP-DEP-NEXT: [[TMP37:%.*]] = mul i64 [[OFF]], 72
+; CHECK-LOOP-DEP-NEXT: [[TMP16:%.*]] = inttoptr i64 [[TMP37]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK25:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP16]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT26:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK25]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT27:%.*]] = xor i1 [[NO_CONFLICT26]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX28:%.*]] = or i1 [[CONFLICT_RDX24]], [[IS_CONFLICT27]]
+; CHECK-LOOP-DEP-NEXT: [[TMP42:%.*]] = mul i64 [[OFF]], 80
+; CHECK-LOOP-DEP-NEXT: [[TMP18:%.*]] = inttoptr i64 [[TMP42]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK29:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP18]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT30:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK29]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT31:%.*]] = xor i1 [[NO_CONFLICT30]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX32:%.*]] = or i1 [[CONFLICT_RDX28]], [[IS_CONFLICT31]]
+; CHECK-LOOP-DEP-NEXT: [[TMP47:%.*]] = mul i64 [[OFF]], 88
+; CHECK-LOOP-DEP-NEXT: [[TMP20:%.*]] = inttoptr i64 [[TMP47]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK257:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP20]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT258:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK257]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT259:%.*]] = xor i1 [[NO_CONFLICT258]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX260:%.*]] = or i1 [[CONFLICT_RDX32]], [[IS_CONFLICT259]]
+; CHECK-LOOP-DEP-NEXT: br i1 [[CONFLICT_RDX260]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-LOOP-DEP: vector.ph:
+; CHECK-LOOP-DEP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-LOOP-DEP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-LOOP-DEP-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-LOOP-DEP: vector.body:
+; CHECK-LOOP-DEP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-LOOP-DEP-NEXT: [[TMP167:%.*]] = add nsw i64 [[INDEX]], -5
+; CHECK-LOOP-DEP-NEXT: [[TMP168:%.*]] = add i64 [[TMP167]], [[OFF]]
+; CHECK-LOOP-DEP-NEXT: [[TMP169:%.*]] = getelementptr i64, ptr [[DST:%.*]], i64 [[TMP168]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP169]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP170:%.*]] = add i64 [[TMP167]], [[OFF_MUL_2]]
+; CHECK-LOOP-DEP-NEXT: [[TMP171:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP170]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP171]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP172:%.*]] = add i64 [[TMP167]], [[OFF_MUL_3]]
+; CHECK-LOOP-DEP-NEXT: [[TMP173:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP172]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP173]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP174:%.*]] = add i64 [[TMP167]], [[OFF_MUL_4]]
+; CHECK-LOOP-DEP-NEXT: [[TMP175:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP174]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP175]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP176:%.*]] = add i64 [[TMP167]], [[OFF_MUL_5]]
+; CHECK-LOOP-DEP-NEXT: [[TMP177:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP176]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP177]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP178:%.*]] = add i64 [[TMP167]], [[OFF_MUL_6]]
+; CHECK-LOOP-DEP-NEXT: [[TMP179:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP178]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP179]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP180:%.*]] = add i64 [[TMP167]], [[OFF_MUL_7]]
+; CHECK-LOOP-DEP-NEXT: [[TMP181:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP180]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP181]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP182:%.*]] = add i64 [[TMP167]], [[OFF_MUL_8]]
+; CHECK-LOOP-DEP-NEXT: [[TMP183:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP182]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP183]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP184:%.*]] = add i64 [[TMP167]], [[OFF_MUL_9]]
+; CHECK-LOOP-DEP-NEXT: [[TMP185:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP184]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP185]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP186:%.*]] = add i64 [[TMP167]], [[OFF_MUL_10]]
+; CHECK-LOOP-DEP-NEXT: [[TMP187:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP186]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP187]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP188:%.*]] = add i64 [[TMP167]], [[OFF_MUL_11]]
+; CHECK-LOOP-DEP-NEXT: [[TMP189:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP188]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP189]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP190:%.*]] = add i64 [[TMP167]], [[OFF_MUL_12]]
+; CHECK-LOOP-DEP-NEXT: [[TMP191:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP190]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> zeroinitializer, ptr [[TMP191]], align 8
+; CHECK-LOOP-DEP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-LOOP-DEP-NEXT: [[TMP192:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-LOOP-DEP-NEXT: br i1 [[TMP192]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-LOOP-DEP: middle.block:
+; CHECK-LOOP-DEP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-LOOP-DEP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-LOOP-DEP: scalar.ph:
+; CHECK-LOOP-DEP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-LOOP-DEP-NEXT: br label [[LOOP:%.*]]
+; CHECK-LOOP-DEP: loop:
+; CHECK-LOOP-DEP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LOOP-DEP-NEXT: [[IV_SUB_5:%.*]] = add nsw i64 [[IV]], -5
+; CHECK-LOOP-DEP-NEXT: [[IDX_1:%.*]] = add i64 [[IV_SUB_5]], [[OFF]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_1:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_1]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_1]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_2:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_2]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_2:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_2]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_2]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_3:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_3]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_3:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_3]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_3]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_4:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_4]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_4:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_4]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_4]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_5:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_5]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_5:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_5]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_5]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_6:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_6]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_6:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_6]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_6]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_7:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_7]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_7:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_7]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_7]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_8:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_8]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_8:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_8]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_8]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_9:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_9]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_9:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_9]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_9]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_10:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_10]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_10:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_10]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_10]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_11:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_11]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_11:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_11]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_11]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IDX_12:%.*]] = add i64 [[IV_SUB_5]], [[OFF_MUL_12]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_12:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IDX_12]]
+; CHECK-LOOP-DEP-NEXT: store double 0.000000e+00, ptr [[GEP_12]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-LOOP-DEP-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-LOOP-DEP-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-LOOP-DEP: exit:
+; CHECK-LOOP-DEP-NEXT: ret void
+;
entry:
%off.mul.2 = shl i64 %off, 1
%off.mul.3 = mul i64 %off, 3
@@ -259,6 +438,60 @@ define void @check_creation_order(ptr %a, ptr %b, i32 %m) {
; CHECK: exit:
; CHECK-NEXT: ret void
;
+; CHECK-LOOP-DEP-LABEL: @check_creation_order(
+; CHECK-LOOP-DEP-NEXT: entry:
+; CHECK-LOOP-DEP-NEXT: [[A1:%.*]] = ptrtoint ptr [[A:%.*]] to i64
+; CHECK-LOOP-DEP-NEXT: [[A2:%.*]] = ptrtoint ptr [[A3:%.*]] to i64
+; CHECK-LOOP-DEP-NEXT: [[M_EXT:%.*]] = sext i32 [[M:%.*]] to i64
+; CHECK-LOOP-DEP-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr double, ptr [[A3]], i64 [[M_EXT]]
+; CHECK-LOOP-DEP-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-LOOP-DEP: vector.memcheck:
+; CHECK-LOOP-DEP-NEXT: [[TMP1:%.*]] = mul nsw i64 [[M_EXT]], -8
+; CHECK-LOOP-DEP-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP2]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT:%.*]] = xor i1 [[NO_CONFLICT]], true
+; CHECK-LOOP-DEP-NEXT: [[TMP8:%.*]] = sub i64 [[A2]], [[A1]]
+; CHECK-LOOP-DEP-NEXT: [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-LOOP-DEP-NEXT: [[LOOP_DEP_MASK3:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr null, ptr [[TMP9]], i64 8)
+; CHECK-LOOP-DEP-NEXT: [[NO_CONFLICT4:%.*]] = extractelement <4 x i1> [[LOOP_DEP_MASK3]], i64 3
+; CHECK-LOOP-DEP-NEXT: [[IS_CONFLICT5:%.*]] = xor i1 [[NO_CONFLICT4]], true
+; CHECK-LOOP-DEP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[IS_CONFLICT]], [[IS_CONFLICT5]]
+; CHECK-LOOP-DEP-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-LOOP-DEP: vector.ph:
+; CHECK-LOOP-DEP-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-LOOP-DEP: vector.body:
+; CHECK-LOOP-DEP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-LOOP-DEP-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[INVARIANT_GEP]], i64 [[INDEX]]
+; CHECK-LOOP-DEP-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP3]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDEX]]
+; CHECK-LOOP-DEP-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
+; CHECK-LOOP-DEP-NEXT: [[TMP5:%.*]] = fadd <4 x double> [[WIDE_LOAD]], [[WIDE_LOAD6]]
+; CHECK-LOOP-DEP-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[A3]], i64 [[INDEX]]
+; CHECK-LOOP-DEP-NEXT: store <4 x double> [[TMP5]], ptr [[TMP6]], align 8
+; CHECK-LOOP-DEP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-LOOP-DEP-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 31996
+; CHECK-LOOP-DEP-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-LOOP-DEP: middle.block:
+; CHECK-LOOP-DEP-NEXT: br label [[SCALAR_PH]]
+; CHECK-LOOP-DEP: scalar.ph:
+; CHECK-LOOP-DEP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 31996, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-LOOP-DEP-NEXT: br label [[LOOP:%.*]]
+; CHECK-LOOP-DEP: loop:
+; CHECK-LOOP-DEP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LOOP-DEP-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[INVARIANT_GEP]], i64 [[IV]]
+; CHECK-LOOP-DEP-NEXT: [[L_0:%.*]] = load double, ptr [[GEP]], align 8
+; CHECK-LOOP-DEP-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; CHECK-LOOP-DEP-NEXT: [[L_1:%.*]] = load double, ptr [[GEP_B]], align 8
+; CHECK-LOOP-DEP-NEXT: [[ADD3:%.*]] = fadd double [[L_0]], [[L_1]]
+; CHECK-LOOP-DEP-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr [[A3]], i64 [[IV]]
+; CHECK-LOOP-DEP-NEXT: store double [[ADD3]], ptr [[GEP_A]], align 8
+; CHECK-LOOP-DEP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-LOOP-DEP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 31999
+; CHECK-LOOP-DEP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-LOOP-DEP: exit:
+; CHECK-LOOP-DEP-NEXT: ret void
+;
entry:
%m.ext = sext i32 %m to i64
%invariant.gep = getelementptr double, ptr %a, i64 %m.ext
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index 68327a0e5e60b..134835a717f67 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -24,11 +24,11 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-VF4UF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
; CHECK-VF4UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK-VF4UF1: [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF1-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
+; CHECK-VF4UF1-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF1-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-VF4UF1-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
; CHECK-VF4UF1-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-VF4UF1-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
-; CHECK-VF4UF1-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF1-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP7]]
; CHECK-VF4UF1-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VF4UF1: [[VECTOR_PH]]:
@@ -87,11 +87,11 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-VF4UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
; CHECK-VF4UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK-VF4UF2: [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF2-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
+; CHECK-VF4UF2-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF2-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-VF4UF2-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
; CHECK-VF4UF2-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 8
-; CHECK-VF4UF2-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
-; CHECK-VF4UF2-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP7]]
; CHECK-VF4UF2-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VF4UF2: [[VECTOR_PH]]:
>From f2126aaf5af7ef6a43f6758e69ee81fab01e4f97 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 10 Feb 2026 16:36:12 +0000
Subject: [PATCH 4/4] Rm
---
.../LoopVectorize/AArch64/sve2-histcnt.ll | 39 -------------------
1 file changed, 39 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 99ead5d68034e..18dba8b0395b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -269,45 +269,6 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N_VEC]], 0
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP9]]
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
-; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 3
-; CHECK-NEXT: [[DOTNOT1:%.*]] = sub nsw i64 0, [[TMP12]]
-; CHECK-NEXT: [[N_VEC3:%.*]] = and i64 [[N]], [[DOTNOT1]]
-; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP14]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[TMP15]], i8 1, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP12]]
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
-; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
-; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
-; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
-; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
-; CHECK-NEXT: [[L_BUCKET:%.*]] = load i8, ptr [[GEP_BUCKET]], align 4
-; CHECK-NEXT: [[INC:%.*]] = add nsw i8 [[L_BUCKET]], 1
-; CHECK-NEXT: store i8 [[INC]], ptr [[GEP_BUCKET]], align 4
-; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK: for.exit:
-; CHECK-NEXT: ret void
;
entry:
br label %for.body
More information about the llvm-commits
mailing list