[llvm] [LV] Convert scatter stores with constant stride into strided stores (PR #206896)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 23:53:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
Author: Mel Chen (Mel-Chen)
<details>
<summary>Changes</summary>
Following up on #<!-- -->147297, this patch converts scatter accesses with a constant stride into strided stores within convertToStridedAccesses.
Fixed #<!-- -->206662
---
Patch is 219.64 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/206896.diff
15 Files Affected:
- (modified) llvm/lib/Analysis/VectorUtils.cpp (+4)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+2-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+16-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+13-6)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+50-19)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll (+35-63)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll (+18-41)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll (+49-32)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll (+7-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll (+402-278)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll (+8-20)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll (+58-100)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll (+156-45)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll (+9-15)
``````````diff
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 193fb6720cf60..a33f17bf00755 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -171,6 +171,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
return ScalarOpdIdx == 2 || ScalarOpdIdx == 4;
case Intrinsic::experimental_vp_strided_load:
return ScalarOpdIdx == 0 || ScalarOpdIdx == 1;
+ case Intrinsic::experimental_vp_strided_store:
+ return ScalarOpdIdx == 1 || ScalarOpdIdx == 2;
case Intrinsic::loop_dependence_war_mask:
return true;
default:
@@ -213,6 +215,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
return OpdIdx == -1 || OpdIdx == 1;
case Intrinsic::experimental_vp_strided_load:
return OpdIdx == -1 || OpdIdx == 0 || OpdIdx == 1;
+ case Intrinsic::experimental_vp_strided_store:
+ return OpdIdx == 0 || OpdIdx == 1 || OpdIdx == 2;
default:
return OpdIdx == -1;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eaf9d1433aff7..78fb1110620cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2056,7 +2056,6 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
Align Alignment;
public:
- // TODO: support StoreInst for strided store
VPWidenMemIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
Align Alignment, const VPIRMetadata &MD = {},
@@ -2065,7 +2064,8 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
VectorIntrinsicID, CallArguments, Ty, {}, MD,
DL),
Alignment(Alignment) {
- assert(VectorIntrinsicID == Intrinsic::experimental_vp_strided_load &&
+ assert((VectorIntrinsicID == Intrinsic::experimental_vp_strided_load ||
+ VectorIntrinsicID == Intrinsic::experimental_vp_strided_store) &&
"Unexpected intrinsic");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index e2e08c50d06e8..7099355eb830b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1017,7 +1017,7 @@ struct IntrinsicID_match {
/// them with lower arity matchers. Here's some convenient typedefs for up to
/// several arguments, and more can be added as needed
template <typename T0 = void, typename T1 = void, typename T2 = void,
- typename T3 = void>
+ typename T3 = void, typename T4 = void>
struct m_Intrinsic_Ty;
template <typename T0> struct m_Intrinsic_Ty<T0> {
using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
@@ -1032,10 +1032,15 @@ struct m_Intrinsic_Ty<T0, T1, T2> {
Argument_match<T2>>;
};
template <typename T0, typename T1, typename T2, typename T3>
-struct m_Intrinsic_Ty {
+struct m_Intrinsic_Ty<T0, T1, T2, T3> {
using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
Argument_match<T3>>;
};
+template <typename T0, typename T1, typename T2, typename T3, typename T4>
+struct m_Intrinsic_Ty {
+ using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty,
+ Argument_match<T4>>;
+};
/// Match intrinsic calls like this:
/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
@@ -1072,6 +1077,15 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
}
+template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
+ typename T3, typename T4>
+inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty
+m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
+ const T4 &Op4) {
+ return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3),
+ m_Argument<4>(Op4));
+}
+
template <Intrinsic::ID IntrID, typename... T>
inline auto m_WidenIntrinsic(const T &...Ops) {
return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f45b9e4f6c35b..17a4073cecf85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2351,9 +2351,13 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
CallInst *MemI = createVectorCall(State);
+ auto PtrPos = VPIntrinsic::getMemoryPointerParamPos(getVectorIntrinsicID());
+ assert(PtrPos.has_value() &&
+ "Expected a memory intrinsic with a valid pointer position");
MemI->addParamAttr(
- 0, Attribute::getWithAlignment(MemI->getContext(), Alignment));
- State.set(this, MemI);
+ *PtrPos, Attribute::getWithAlignment(MemI->getContext(), Alignment));
+ if (!MemI->getType()->isVoidTy())
+ State.set(this, MemI);
}
InstructionCost VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
@@ -2367,10 +2371,13 @@ InstructionCost VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
- Type *Ty = toVectorTy(getScalarType(), VF);
- return computeMemIntrinsicCost(getVectorIntrinsicID(), Ty,
- !match(getOperand(2), m_True()), Alignment,
- Ctx);
+ Type *DataTy = getScalarType();
+ if (DataTy->isVoidTy())
+ DataTy = getOperand(0)->getScalarType();
+ Type *Ty = toVectorTy(DataTy, VF);
+ return computeMemIntrinsicCost(
+ getVectorIntrinsicID(), Ty,
+ !match(getOperand(getNumOperands() - 2), m_True()), Alignment, Ctx);
}
void VPHistogramRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ceb7c38ca3e29..bb5672c9cbdda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3105,6 +3105,18 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
SpliceR, EVL, Mask);
}
+ if (match(&CurRecipe, m_Intrinsic<Intrinsic::experimental_vp_strided_store>(
+ m_VPValue(StoredVal), m_VPValue(Addr),
+ m_VPValue(Stride), m_RemoveMask(HeaderMask, Mask),
+ m_TruncOrSelf(m_Specific(&Plan->getVF()))))) {
+ if (!Mask)
+ Mask = Plan->getTrue();
+ auto *NewStore = cast<VPWidenMemIntrinsicRecipe>(&CurRecipe)->clone();
+ NewStore->setOperand(3, Mask);
+ NewStore->setOperand(4, &EVL);
+ return NewStore;
+ }
+
if (auto *Rdx = dyn_cast<VPReductionRecipe>(&CurRecipe))
if (Rdx->isConditional() &&
match(Rdx->getCondOp(), m_RemoveMask(HeaderMask, Mask)))
@@ -7525,16 +7537,15 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(VectorLoop->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *LoadR = dyn_cast<VPWidenLoadRecipe>(&R);
- // TODO: Support strided store.
+ auto *MemR = dyn_cast<VPWidenMemoryRecipe>(&R);
// TODO: Transform reverse access into strided access with -1 stride.
// TODO: Transform gather/scatter with uniform address into strided access
// with 0 stride.
// TODO: Transform interleave access into multiple strided accesses.
- if (!LoadR || LoadR->isConsecutive())
+ if (!MemR || MemR->isConsecutive())
continue;
- auto *Ptr = dyn_cast<VPWidenGEPRecipe>(LoadR->getAddr());
+ auto *Ptr = dyn_cast<VPWidenGEPRecipe>(MemR->getAddr());
if (!Ptr)
continue;
@@ -7549,17 +7560,30 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
m_SpecificLoop(&L))))
continue;
- Type *LoadTy = LoadR->getScalarType();
- Align Alignment = LoadR->getAlign();
+ VPValue *StoredValue = nullptr;
+ Type *DataTy = nullptr;
+ if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(&R)) {
+ StoredValue = StoreR->getStoredValue();
+ DataTy = StoredValue->getScalarType();
+ } else if (auto *LoadR = dyn_cast<VPWidenLoadRecipe>(&R)) {
+ DataTy = LoadR->getScalarType();
+ } else {
+ continue;
+ }
+
+ Intrinsic::ID IntrinID = StoredValue
+ ? Intrinsic::experimental_vp_strided_store
+ : Intrinsic::experimental_vp_strided_load;
+
+ Align Alignment = MemR->getAlign();
auto IsProfitable = [&](ElementCount VF) {
- Type *DataTy = toVectorTy(LoadTy, VF);
- if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment))
+ Type *VectorTy = toVectorTy(DataTy, VF);
+ if (!Ctx.TTI.isLegalStridedLoadStore(VectorTy, Alignment))
return false;
- const InstructionCost CurrentCost = LoadR->computeCost(VF, Ctx);
+ const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
const InstructionCost StridedLoadStoreCost =
VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
- Intrinsic::experimental_vp_strided_load, DataTy,
- LoadR->isMasked(), Alignment, Ctx);
+ IntrinID, VectorTy, MemR->isMasked(), Alignment, Ctx);
return StridedLoadStoreCost < CurrentCost;
};
@@ -7571,7 +7595,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
// not counted during precomputeCosts.
// TODO: Remove once the legacy exit cost computation is retired.
for (ElementCount VF : Range)
- Ctx.invalidateWideningDecision(&LoadR->getIngredient(), VF);
+ Ctx.invalidateWideningDecision(&MemR->getIngredient(), VF);
// Get VF as i32 for the vector length operand.
if (!I32VF) {
@@ -7581,7 +7605,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Plan.getVF().getScalarType(), DebugLoc::getUnknown());
}
- VPBuilder Builder(LoadR);
+ VPBuilder Builder(&R);
// Create the base pointer of strided access.
// TODO: reuse VPDerivedIVRecipe for base pointer computation when it
// supports a general VPValue as the start value.
@@ -7607,14 +7631,21 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes,
Ptr->getGEPNoWrapFlags(), Ptr->getDebugLoc());
- VPValue *Mask = LoadR->getMask();
+ VPValue *Mask = MemR->getMask();
if (!Mask)
Mask = Plan.getTrue();
- auto *StridedLoad = Builder.createWidenMemIntrinsic(
- Intrinsic::experimental_vp_strided_load,
- {NewPtr, StrideInBytes, Mask, I32VF}, LoadTy, Alignment, *LoadR,
- LoadR->getDebugLoc());
- LoadR->replaceAllUsesWith(StridedLoad);
+ SmallVector<VPValue *, 4> Ops;
+ if (StoredValue)
+ Ops.push_back(StoredValue);
+ Ops.append({NewPtr, StrideInBytes, Mask, I32VF});
+
+ auto *StridedR = Builder.createWidenMemIntrinsic(
+ IntrinID, Ops,
+ StoredValue ? Type::getVoidTy(Plan.getContext()) : DataTy, Alignment,
+ *MemR, R.getDebugLoc());
+ if (!StoredValue)
+ cast<VPWidenLoadRecipe>(&R)->replaceAllUsesWith(StridedR);
+ R.eraseFromParent();
}
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index 28b418b5c0cdf..7ee772d8f1b7c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -13,21 +13,17 @@ define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <vscale x 8 x i64> [[TMP11]], splat (i64 3)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> zeroinitializer, ptr align 2 [[TMP5]], i64 6, <vscale x 8 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT: [[TMP16:%.*]] = mul nsw i64 3, [[TMP17]]
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP20]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP17]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP17]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -67,21 +63,17 @@ define void @block_with_dead_inst_2(ptr %src) #0 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <vscale x 8 x i64> [[TMP5]], splat (i64 3)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 333, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> zeroinitializer, ptr align 2 [[TMP2]], i64 6, <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP9]] to i64
-; CHECK-NEXT: [[TMP13:%.*]] = mul nsw i64 3, [[TMP7]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -121,21 +113,17 @@ define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <vscale x 8 x i64> [[TMP5]], splat (i64 3)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 333, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> zeroinitializer, ptr align 2 [[TMP2]], i64 6, <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP9]] to i64
-; CHECK-NEXT: [[TMP13:%.*]] = mul nsw i64 3, [[TMP7]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -182,21 +170,17 @@ define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <vscale x 8 x i64> [[TMP11]], splat (i64 3)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> zeroinitializer, ptr a...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/206896
More information about the llvm-commits
mailing list