[llvm] [LV] isScalarWithPredication to check for interleaved accesses (PR #194579)
Gaëtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 02:22:54 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/194579
>From 9d29197bb1ac0182d6d4ac8188515f2463f4b30d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 29 Apr 2026 15:45:20 +0000
Subject: [PATCH 1/2] [LV] Add test showing lack of gather/scatter can prevent
if-convert
This introduces a new force-target-supports-masked-memory-ops CLI option
for testing, as well a new isLegalMaskedLoadOrStore() helper.
---
.../Vectorize/LoopVectorizationLegality.h | 5 +-
.../Vectorize/LoopVectorizationLegality.cpp | 9 +-
.../Vectorize/LoopVectorizationPlanner.cpp | 34 +++--
.../Vectorize/LoopVectorizationPlanner.h | 13 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 20 +--
.../LoopVectorize/if-conversion-scalable.ll | 120 ++++++++++++++++++
6 files changed, 156 insertions(+), 45 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..b5d63fc289ad0 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -389,13 +389,16 @@ class LoopVectorizationLegality {
/// when the last index of the GEP is the induction variable, or that the
/// pointer itself is an induction variable.
/// This check allows us to vectorize A[idx] into a wide load/store.
+ /// If InterleaveFactor>1, this checks whether the pointer is consecutive
+ /// for the whole interleave group.
/// Returns:
/// 0 - Stride is unknown or non-consecutive.
/// 1 - Address is consecutive.
/// -1 - Address is consecutive, and decreasing.
/// NOTE: This method must only be used before modifying the original scalar
/// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
- int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
+ int isConsecutivePtr(Type *AccessTy, Value *Ptr,
+ unsigned InterleaveFactor = 1) const;
/// Returns true if \p V is invariant across all loop iterations according to
/// SCEV.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index b2d0c6bb11202..5821ad57d887a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -476,8 +476,9 @@ static bool storeToSameAddress(ScalarEvolution *SE, StoreInst *A,
return SE->getSCEV(APtr) == SE->getSCEV(BPtr);
}
-int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
- Value *Ptr) const {
+int LoopVectorizationLegality::isConsecutivePtr(
+ Type *AccessTy, Value *Ptr, unsigned InterleaveFactor) const {
+ assert(InterleaveFactor >= 1);
// FIXME: Currently, the set of symbolic strides is sometimes queried before
// it's collected. This happens from canVectorizeWithIfConvert, when the
// pointer is checked to reference consecutive elements suitable for a
@@ -488,8 +489,8 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
int Stride = getPtrStride(PSE, AccessTy, Ptr, TheLoop, *DT, Strides,
AllowRuntimeSCEVChecks, false)
.value_or(0);
- if (Stride == 1 || Stride == -1)
- return Stride;
+ if (Stride == int(InterleaveFactor) || Stride == -int(InterleaveFactor))
+ return Stride / int(InterleaveFactor);
return 0;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 0e847f4767a8b..5970d6a82e868 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -59,20 +59,25 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
cl::desc("Assume the target supports masked memory operations (used for "
"testing)."));
-bool VFSelectionContext::isLegalMaskedStore(Type *DataType, Value *Ptr,
- Align Alignment,
- unsigned AddressSpace) const {
- return Legal->isConsecutivePtr(DataType, Ptr) &&
- (ForceTargetSupportsMaskedMemoryOps ||
- TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace));
-}
+static cl::opt<bool> ForceTargetSupportsGatherScatterOps(
+ "force-target-supports-gather-scatter-ops", cl::init(false), cl::Hidden,
+ cl::desc("Assume the target supports gather/scatter operations (used for "
+ "testing)."));
+
+bool VFSelectionContext::isLegalMaskedLoadOrStore(
+ Instruction *I, ElementCount VF, unsigned InterleaveFactor) const {
+ assert(isa<LoadInst>(I) || isa<StoreInst>(I));
+ auto *Ptr = getLoadStorePointerOperand(I);
+ auto *Ty = getLoadStoreType(I);
+ const unsigned AS = getLoadStoreAddressSpace(I);
+ const Align Alignment = getLoadStoreAlignment(I);
+
+ if (!Legal->isConsecutivePtr(Ty, Ptr, InterleaveFactor))
+ return false;
-bool VFSelectionContext::isLegalMaskedLoad(Type *DataType, Value *Ptr,
- Align Alignment,
- unsigned AddressSpace) const {
- return Legal->isConsecutivePtr(DataType, Ptr) &&
- (ForceTargetSupportsMaskedMemoryOps ||
- TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace));
+ return ForceTargetSupportsMaskedMemoryOps ||
+ (isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
+ : TTI.isLegalMaskedStore(Ty, Alignment, AS));
}
bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
@@ -85,7 +90,8 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
Align Align = getLoadStoreAlignment(V);
if (VF.isVector())
Ty = VectorType::get(Ty, VF);
- return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
+ return ForceTargetSupportsGatherScatterOps ||
+ (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b07a17f2d8baa..4ac0286408f89 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -650,15 +650,10 @@ class VFSelectionContext {
/// of FP operations.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const;
- /// Returns true if the target machine supports masked store operation
- /// for the given \p DataType and kind of access to \p Ptr.
- bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
- unsigned AddressSpace) const;
-
- /// Returns true if the target machine supports masked load operation
- /// for the given \p DataType and kind of access to \p Ptr.
- bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
- unsigned AddressSpace) const;
+ /// Returns true if the target machine supports masked load or store
+ /// operations for the given \p VF and \p InterleaveFactor.
+ bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF,
+ unsigned InterleaveFactor = 1) const;
/// Returns true if the target machine can represent \p V as a masked gather
/// or scatter operation.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6e7a17c8b59d5..90844b56e5bea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2465,18 +2465,8 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
return getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize;
case Instruction::Load:
case Instruction::Store: {
- auto *Ptr = getLoadStorePointerOperand(I);
- auto *Ty = getLoadStoreType(I);
- unsigned AS = getLoadStoreAddressSpace(I);
- Type *VTy = Ty;
- if (VF.isVector())
- VTy = VectorType::get(Ty, VF);
- const Align Alignment = getLoadStoreAlignment(I);
- return isa<LoadInst>(I)
- ? !(Config.isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
- TTI.isLegalMaskedGather(VTy, Alignment))
- : !(Config.isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
- TTI.isLegalMaskedScatter(VTy, Alignment));
+ return !Config.isLegalMaskedLoadOrStore(I, VF) &&
+ !Config.isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
case Instruction::SDiv:
@@ -2699,11 +2689,7 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
if (VF.isScalable() && NeedsMaskForGaps)
return false;
- auto *Ty = getLoadStoreType(I);
- const Align Alignment = getLoadStoreAlignment(I);
- unsigned AS = getLoadStoreAddressSpace(I);
- return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
- : TTI.isLegalMaskedStore(Ty, Alignment, AS);
+ return Config.isLegalMaskedLoadOrStore(I, VF, InterleaveFactor);
}
bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
new file mode 100644
index 0000000000000..1e0d7eafc9096
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
+
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
+; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
+; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -force-target-supports-gather-scatter-ops \
+; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-ENABLED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
+; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
+; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses \
+; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-DISABLED
+
+; Test whether the legality of gather/scatter has an impact on if-conversion
+; for scalable VFs. In either case, the example below should be vectorized with
+; VF = vscale x 8 and a masked deinterleaved load.
+
+define void @conv_interleaved_loads_load(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %flags, i64 %cond) {
+; CHECK-GATHER-ENABLED-LABEL: @conv_interleaved_loads_load(
+; CHECK-GATHER-ENABLED-NEXT: entry:
+; CHECK-GATHER-ENABLED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-GATHER-ENABLED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-GATHER-ENABLED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-GATHER-ENABLED: vector.ph:
+; CHECK-GATHER-ENABLED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-GATHER-ENABLED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-GATHER-ENABLED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-GATHER-ENABLED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-GATHER-ENABLED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-GATHER-ENABLED: vector.body:
+; CHECK-GATHER-ENABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2
+; CHECK-GATHER-ENABLED-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; CHECK-GATHER-ENABLED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP7:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-GATHER-ENABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP6]], <vscale x 8 x i1> [[TMP6]])
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison)
+; CHECK-GATHER-ENABLED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]])
+; CHECK-GATHER-ENABLED-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-GATHER-ENABLED-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-GATHER-ENABLED-NEXT: [[TMP10:%.*]] = add <vscale x 8 x i16> [[TMP8]], [[TMP9]]
+; CHECK-GATHER-ENABLED-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> [[TMP10]], <vscale x 8 x i16> zeroinitializer
+; CHECK-GATHER-ENABLED-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
+; CHECK-GATHER-ENABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-GATHER-ENABLED: middle.block:
+; CHECK-GATHER-ENABLED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-GATHER-ENABLED: scalar.ph:
+;
+; CHECK-GATHER-DISABLED-LABEL: @conv_interleaved_loads_load(
+; CHECK-GATHER-DISABLED-NEXT: entry:
+; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-GATHER-DISABLED: vector.ph:
+; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-GATHER-DISABLED: vector.body:
+; CHECK-GATHER-DISABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
+; CHECK-GATHER-DISABLED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; CHECK-GATHER-DISABLED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[TMP3]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i16> poison)
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-GATHER-DISABLED-NEXT: [[TMP4:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[STRIDED_VEC2]]
+; CHECK-GATHER-DISABLED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP4]], <8 x i16> zeroinitializer
+; CHECK-GATHER-DISABLED-NEXT: [[TMP5:%.*]] = add <8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: store <8 x i16> [[TMP5]], ptr [[TMP6]], align 2
+; CHECK-GATHER-DISABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-GATHER-DISABLED-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-GATHER-DISABLED-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-GATHER-DISABLED: middle.block:
+; CHECK-GATHER-DISABLED-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-GATHER-DISABLED: for.cond.cleanup:
+; CHECK-GATHER-DISABLED-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %indvars.iv
+ %0 = load i16, ptr %arrayidx0, align 2
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds [2 x i16], ptr %c, i64 %indvars.iv, i64 0
+ %pred.1 = load i16, ptr %arrayidx1, align 2
+ %arrayidx2 = getelementptr inbounds [2 x i16], ptr %c, i64 %indvars.iv, i64 1
+ %pred.2 = load i16, ptr %arrayidx2, align 2
+ %pred.sum = add i16 %pred.1, %pred.2
+ br label %for.inc
+
+for.inc:
+ %1 = phi i16 [ zeroinitializer, %for.body ], [ %pred.sum, %if.then ]
+ %result = add i16 %0, %1
+ %arrayidx3 = getelementptr inbounds i16, ptr %a, i64 %indvars.iv
+ store i16 %result, ptr %arrayidx3, align 2
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
>From 48e90956e5ec2ea5784717eec9f442e7230e1fad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 27 Apr 2026 12:03:51 +0000
Subject: [PATCH 2/2] [LV] isScalarWithPredication to check for interleaved
accesses
isScalarWithPredication used to assume that predicated interleaved
accesses were supported iff the target supports predicated
gather/scatters.
The change is mostly NFC as targets enabling masked interleaved accesses
(AArch64, RISCV) also enable gather/scatter.
However, there are subtle problems with isScalarWithPredication not considering masked interleaved ld/st:
- Instructions can get marked as isProfitableToScalarize (for fixed VFs) even though they will actually be widened because CM_Interleave takes precedence over isProfitableToScalarize. See tryToWidenMemory. This means we add the wrong cost: precomputeCosts adds the scalarization cost, whereas the widen recipe actually gets executed.
- Having any instruction trigger isScalarWithPredication means its parent BB gets added to PredicatedBBsAfterVectorization. The consequence is that this disables if-conversion for scalable VFs: precomputeCosts() thinks the conditional branch is still there, which gives Invalid costs for scalable VFs.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 15 ++-
.../CostModel/masked-interleaved-store-i16.ll | 6 +-
.../LoopVectorize/if-conversion-scalable.ll | 114 +++++++-----------
3 files changed, 56 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 90844b56e5bea..585d6dc494f16 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2465,8 +2465,13 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
return getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize;
case Instruction::Load:
case Instruction::Store: {
- return !Config.isLegalMaskedLoadOrStore(I, VF) &&
- !Config.isLegalGatherOrScatter(I, VF);
+ LoopVectorizationCostModel::InstWidening WidenKind =
+ getWideningDecision(I, VF);
+ assert(WidenKind != CM_Unknown);
+ // According to the legacy cost model, a masked access is never considered
+ // profitable for scalarization when gather/scatter is available, even if
+ // the chosen widening is CM_Scalarize.
+ return WidenKind == CM_Scalarize && !Config.isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
case Instruction::SDiv:
@@ -2706,7 +2711,7 @@ bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
// If the instruction is a store located in a predicated block, it will be
// scalarized.
- if (isScalarWithPredication(I, VF))
+ if (isPredicatedInst(I) && !Config.isLegalMaskedLoadOrStore(I, VF))
return false;
// If the instruction's allocated size doesn't equal it's type size, it
@@ -4807,7 +4812,9 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
// predicated uniform stores. Today they are treated as any other
// predicated store (see added test cases in
// invariant-store-vectorization.ll).
- if (isa<StoreInst>(&I) && isScalarWithPredication(&I, VF))
+ if (isa<StoreInst>(&I) && isPredicatedInst(&I) &&
+ !Config.isLegalMaskedLoadOrStore(&I, VF) &&
+ !Config.isLegalGatherOrScatter(&I, VF))
NumPredStores++;
if (Legal->isUniformMemOp(I, VF)) {
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
index 398f553059297..14a68cef22e2d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
@@ -161,8 +161,10 @@ define void @test(ptr noalias nocapture %points, ptr noalias nocapture readonly
; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %0, ptr %arrayidx6, align 2
; ENABLED_MASKED_STRIDED: Cost of 2 for VF 2: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
; ENABLED_MASKED_STRIDED: Cost of 4 for VF 4: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
-; ENABLED_MASKED_STRIDED: Cost of 8 for VF 8: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
-; ENABLED_MASKED_STRIDED: Cost of 16 for VF 16: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
+; ENABLED_MASKED_STRIDED: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%arrayidx6>, ir<%cmp1>
+; ENABLED_MASKED_STRIDED: store ir<%0> to index 0
+; ENABLED_MASKED_STRIDED: Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%arrayidx6>, ir<%cmp1>
+; ENABLED_MASKED_STRIDED: store ir<%0> to index 0
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
index 1e0d7eafc9096..87c0e408c8ef4 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
@@ -3,88 +3,56 @@
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -force-target-supports-gather-scatter-ops \
-; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-ENABLED
+; RUN: -S < %s | FileCheck %s
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses \
-; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-DISABLED
+; RUN: -S < %s | FileCheck %s
; Test whether the legality of gather/scatter has an impact on if-conversion
; for scalable VFs. In either case, the example below should be vectorized with
; VF = vscale x 8 and a masked deinterleaved load.
define void @conv_interleaved_loads_load(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %flags, i64 %cond) {
-; CHECK-GATHER-ENABLED-LABEL: @conv_interleaved_loads_load(
-; CHECK-GATHER-ENABLED-NEXT: entry:
-; CHECK-GATHER-ENABLED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-GATHER-ENABLED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-GATHER-ENABLED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-GATHER-ENABLED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-GATHER-ENABLED: vector.ph:
-; CHECK-GATHER-ENABLED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-GATHER-ENABLED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
-; CHECK-GATHER-ENABLED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-GATHER-ENABLED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND:%.*]], i64 0
-; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-GATHER-ENABLED-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-GATHER-ENABLED: vector.body:
-; CHECK-GATHER-ENABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-GATHER-ENABLED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2
-; CHECK-GATHER-ENABLED-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
-; CHECK-GATHER-ENABLED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; CHECK-GATHER-ENABLED-NEXT: [[TMP7:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
-; CHECK-GATHER-ENABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP6]], <vscale x 8 x i1> [[TMP6]])
-; CHECK-GATHER-ENABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison)
-; CHECK-GATHER-ENABLED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]])
-; CHECK-GATHER-ENABLED-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
-; CHECK-GATHER-ENABLED-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
-; CHECK-GATHER-ENABLED-NEXT: [[TMP10:%.*]] = add <vscale x 8 x i16> [[TMP8]], [[TMP9]]
-; CHECK-GATHER-ENABLED-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> [[TMP10]], <vscale x 8 x i16> zeroinitializer
-; CHECK-GATHER-ENABLED-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
-; CHECK-GATHER-ENABLED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-ENABLED-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
-; CHECK-GATHER-ENABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-GATHER-ENABLED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-GATHER-ENABLED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-GATHER-ENABLED: middle.block:
-; CHECK-GATHER-ENABLED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-GATHER-ENABLED-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; CHECK-GATHER-ENABLED: scalar.ph:
-;
-; CHECK-GATHER-DISABLED-LABEL: @conv_interleaved_loads_load(
-; CHECK-GATHER-DISABLED-NEXT: entry:
-; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-GATHER-DISABLED: vector.ph:
-; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[COND:%.*]], i64 0
-; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-GATHER-DISABLED: vector.body:
-; CHECK-GATHER-DISABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-GATHER-DISABLED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
-; CHECK-GATHER-DISABLED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; CHECK-GATHER-DISABLED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; CHECK-GATHER-DISABLED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
-; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
-; CHECK-GATHER-DISABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[TMP3]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i16> poison)
-; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-GATHER-DISABLED-NEXT: [[TMP4:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[STRIDED_VEC2]]
-; CHECK-GATHER-DISABLED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP4]], <8 x i16> zeroinitializer
-; CHECK-GATHER-DISABLED-NEXT: [[TMP5:%.*]] = add <8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
-; CHECK-GATHER-DISABLED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-GATHER-DISABLED-NEXT: store <8 x i16> [[TMP5]], ptr [[TMP6]], align 2
-; CHECK-GATHER-DISABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-GATHER-DISABLED-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-GATHER-DISABLED-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-GATHER-DISABLED: middle.block:
-; CHECK-GATHER-DISABLED-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
-; CHECK-GATHER-DISABLED: for.cond.cleanup:
-; CHECK-GATHER-DISABLED-NEXT: ret void
+; CHECK-LABEL: @conv_interleaved_loads_load(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP6]], <vscale x 8 x i1> [[TMP6]])
+; CHECK-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison)
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 8 x i16> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> [[TMP10]], <vscale x 8 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
;
entry:
br label %for.body
More information about the llvm-commits
mailing list