[llvm] [VPlan] Relax step to loop-inv in convToStridedAccess (PR #214476)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 05:40:52 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Ramkumar Ramachandra (artagnon)
<details>
<summary>Changes</summary>
Observe that the logic in convertToStridedAccesses is essentially finding the stride of a pointer, albeit in a poor and incomplete manner. Introduce vputils::getPointerStride as an authoritative helper for getting the stride of a pointer in VPlan, drawing from very similar existing logic in LoopAccessAnalysis, and use it to clean up and generalize the AddRec step in convertToStridedAccesses to a multipler by a loop-invariant SCEVUnknown. This allows us to materialize more strided loads, and has the side-effect of not dropping wrap-flags by virtue of using SE::getPointerBase and SE::removePointerBase.
---
Patch is 42.07 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214476.diff
11 Files Affected:
- (modified) llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h (+24)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+8-14)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+32)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.h (+8)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll (+16-16)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll (+26-20)
``````````diff
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
index b6820fbd8075b..6bfb182513868 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
@@ -80,6 +80,7 @@ inline cst_pred_ty<is_all_ones> m_scev_AllOnes() {
inline auto m_SCEV() { return m_Isa<const SCEV>(); }
inline auto m_SCEVConstant() { return m_Isa<const SCEVConstant>(); }
+inline auto m_SCEVUnknown() { return m_Isa<const SCEVUnknown>(); }
inline auto m_SCEVVScale() { return m_Isa<const SCEVVScale>(); }
/// Match a SCEV, capturing it if we match.
@@ -137,6 +138,18 @@ inline cst_pred_ty<is_specific_signed_cst> m_scev_SpecificSInt(int64_t V) {
return V;
}
+struct match_specific_apint {
+ const APInt &C;
+ match_specific_apint(const APInt &C) : C(C) {}
+ bool match(const SCEV *S) const {
+ return isa<SCEVConstant>(S) &&
+ APInt::isSameValue(C, cast<SCEVConstant>(S)->getAPInt());
+ }
+};
+
+/// Match an SCEV constant with an APInt.
+inline match_specific_apint m_scev_SpecificInt(const APInt &C) { return C; }
+
struct bind_cst_ty {
const APInt *&CR;
@@ -174,6 +187,17 @@ inline SCEVUnaryExpr_match<SCEVTy, Op0_t> m_scev_Unary(const Op0_t &Op0) {
return SCEVUnaryExpr_match<SCEVTy, Op0_t>(Op0);
}
+template <typename Op0_t>
+inline SCEVUnaryExpr_match<SCEVIntegralCastExpr, Op0_t>
+m_scev_IntegralCast(const Op0_t &Op0) {
+ return m_scev_Unary<SCEVIntegralCastExpr>(Op0);
+}
+
+template <typename Op0_t>
+inline auto m_scev_IntegralCastOrSelf(const Op0_t &Op0) {
+ return m_CombineOr(m_scev_IntegralCast(Op0), Op0);
+}
+
template <typename Op0_t>
inline SCEVUnaryExpr_match<SCEVSignExtendExpr, Op0_t>
m_scev_SExt(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 541a3da855992..2309d0027b797 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5779,14 +5779,11 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
VPValue *Ptr = LoadR->getAddr();
// Check if this is a strided access by analyzing the address SCEV for an
// affine addRec.
- const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
- const SCEV *Start;
- const SCEVConstant *Step;
- // TODO: Support non-constant loop invariant stride.
- if (!match(PtrSCEV,
- m_scev_AffineAddRec(m_SCEV(Start), m_SCEVConstant(Step),
- m_SpecificLoop(&L))))
+ auto StrideTup = vputils::getPointerStride(Ptr, PSE, L);
+ if (!StrideTup)
continue;
+ auto [Start, Step, SCEVNW] = *StrideTup;
+ bool HasNUW = any(SCEVNW & SCEVNoWrapFlags::FlagNUW);
Type *LoadTy = LoadR->getScalarType();
Align Alignment = LoadR->getAlign();
@@ -5829,19 +5826,16 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
.tryToExpand(Start);
if (!StartVPV)
StartVPV = VPBuilder(Plan.getEntry()).createExpandSCEV(Start);
- VPValue *StrideInBytes = Plan.getOrAddLiveIn(Step->getValue());
+ VPValue *StrideInBytes = Plan.getConstantInt(Step);
Type *IndexTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
assert(IndexTy == StrideInBytes->getScalarType() &&
"Stride type from SCEV must match the index type");
VPValue *CanIV = Builder.createScalarZExtOrTrunc(
VectorLoop->getCanonicalIV(), IndexTy, DebugLoc::getUnknown());
- auto *AddRecPtr = cast<SCEVAddRecExpr>(PtrSCEV);
auto *Offset = Builder.createOverflowingOp(
- Instruction::Mul, {CanIV, StrideInBytes},
- {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
- GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
- ? GEPNoWrapFlags::noUnsignedWrap()
- : GEPNoWrapFlags::none();
+ Instruction::Mul, {CanIV, StrideInBytes}, {HasNUW, /*HasNSW=*/false});
+ GEPNoWrapFlags NWFlags =
+ HasNUW ? GEPNoWrapFlags::noUnsignedWrap() : GEPNoWrapFlags::none();
VPValue *BasePtr = Builder.createNoWrapPtrAdd(StartVPV, Offset, NWFlags);
// Create a new vector pointer for strided access.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 93b18b31e9e7d..89cdd0fec2122 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -346,6 +346,38 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
return PSE.getPredicatedSCEV(Expr);
}
+std::optional<std::tuple<const SCEV *, APInt, SCEVNoWrapFlags>>
+vputils::getPointerStride(const VPValue *Ptr, PredicatedScalarEvolution &PSE,
+ const Loop &L) {
+ assert(Ptr->getScalarType()->isPointerTy() && "Ptr must be pointer type");
+ ScalarEvolution &SE = *PSE.getSE();
+ const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
+ if (isa<SCEVCouldNotCompute>(PtrSCEV))
+ return std::nullopt;
+ const SCEV *PointerBase = SE.getPointerBase(PtrSCEV);
+ const SCEV *StrideExpr = SE.removePointerBase(PtrSCEV);
+ const APInt *ScalingFactor;
+ APInt One(StrideExpr->getType()->getIntegerBitWidth(), 1);
+ if (!match(StrideExpr,
+ m_scev_Mul(m_scev_APInt(ScalingFactor),
+ m_scev_IntegralCastOrSelf(m_SCEV(StrideExpr)))))
+ ScalingFactor = &One;
+ const SCEV *Start;
+ const SCEV *Step;
+ if (!match(StrideExpr, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
+ m_SpecificLoop(&L))) ||
+ !SE.isLoopInvariant(Step, &L))
+ return std::nullopt;
+ const APInt *StepC;
+ if (match(Step,
+ m_CombineOr(m_scev_APInt(StepC),
+ m_scev_Mul(m_scev_APInt(StepC), m_SCEVUnknown()))) &&
+ StepC->zext(ScalingFactor->getBitWidth()).urem(*ScalingFactor) == 0)
+ return std::make_tuple(SE.getAddExpr(PointerBase, Start), *StepC,
+ cast<SCEVAddRecExpr>(StrideExpr)->getNoWrapFlags());
+ return std::nullopt;
+}
+
bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
const Loop *L) {
// If address is an SCEVAddExpr, we require that all operands must be either
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..66a9e9dc56fdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -46,6 +46,14 @@ const SCEV *getSCEVExprForVPValue(const VPValue *V,
PredicatedScalarEvolution &PSE,
const Loop *L = nullptr);
+/// Digs out a stride for \p Ptr, and returns a tuple of the start, step
+/// constant, and the AddRec's wrap flags. Returns std::nullopt when no valid
+/// stride is found. It is up to the caller to check that the step constant can
+/// actually be used with any possibly-rewritten pointer-offset computations.
+std::optional<std::tuple<const SCEV *, APInt, SCEVNoWrapFlags>>
+getPointerStride(const VPValue *Ptr, PredicatedScalarEvolution &PSE,
+ const Loop &L);
+
/// Returns true if \p Addr is an address SCEV that can be passed to
/// TTI::getAddressComputationCost, i.e. the address SCEV is loop invariant, an
/// affine AddRec (i.e. induction ), or an add expression of such operands or a
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index af995391762d5..050643d2e1b6c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -92,8 +92,8 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i32> [[VEC_IND]] to <vscale x 16 x i64>
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[INDEX]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr nuw i8, ptr [[SRC]], i64 [[TMP6]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 16 x i64> [[TMP9]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index 2957b402e2ceb..37b9aea8d05e1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -191,8 +191,8 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; RV64-NEXT: [[TMP5:%.*]] = mul nuw i64 [[INDEX]], 112
-; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP5]]
+; RV64-NEXT: [[TMP8:%.*]] = mul i64 [[INDEX]], 112
+; RV64-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP8]]
; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; RV64-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV64-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
@@ -246,8 +246,8 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; RV32-NEXT: [[TMP6:%.*]] = trunc i64 [[INDEX]] to i32
-; RV32-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 112
-; RV32-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP2]], i32 [[TMP7]]
+; RV32-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], 112
+; RV32-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP2]], i32 [[TMP7]]
; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP8]], i32 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; RV32-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV32-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 4dcf948d4a23e..b85aeca5c9150 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -47,9 +47,9 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32-NEXT: [[TMP13:%.*]] = getelementptr nuw i8, ptr [[TRIGGER]], i32 [[TMP6]]
; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i32(ptr align 4 [[TMP13]], i32 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
; RV32-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV32-NEXT: [[TMP20:%.*]] = shl i32 [[TMP12]], 8
-; RV32-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[B]], i32 [[TMP20]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i32(ptr align 8 [[TMP25]], i32 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
+; RV32-NEXT: [[TMP15:%.*]] = shl nuw i32 [[TMP12]], 8
+; RV32-NEXT: [[TMP16:%.*]] = getelementptr nuw i8, ptr [[B]], i32 [[TMP15]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i32(ptr align 8 [[TMP16]], i32 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
; RV32-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV32-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV32-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
@@ -117,8 +117,8 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[TRIGGER]], i64 [[TMP5]]
; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP6]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV64-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 8
-; RV64-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
+; RV64-NEXT: [[TMP9:%.*]] = shl nuw i64 [[INDEX]], 8
+; RV64-NEXT: [[TMP11:%.*]] = getelementptr nuw i8, ptr [[B]], i64 [[TMP9]]
; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP11]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
; RV64-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV64-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
index a31bb408dcbc7..ef937b39aaf95 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
@@ -81,31 +81,25 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_PH]]:
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[STRIDE]], i64 0
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT15]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP35:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; NO-UNIT-STRIDE-MV-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 2 x i64> splat (i64 1), [[TMP35]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP19:%.*]] = shl i64 [[STRIDE]], 3
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[P_STRIDED]], i64 [[TMP19]]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
-; NO-UNIT-STRIDE-MV-NEXT: [[STEP_ADD_3:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP26]], i64 0
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT14]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; NO-UNIT-STRIDE-MV-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT16]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[OFFSET_IDX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP40:%.*]] = getelementptr i64, ptr [[P_STRIDED]], <vscale x 2 x i64> [[TMP34]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_LOAD19:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP30]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META0:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP40]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP27:%.*]] = shl i64 [[INDEX]], 3
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP27]]
+; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP28]], i64 8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
-; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; NO-UNIT-STRIDE-MV: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
index 6955e409355e2..771423ed65c51 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
@@ -18,8 +18,8 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw <vscale x 2 x i32> [[VEC_IND]], splat (i32 3)
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i32> [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 6
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP5]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i32 [[INDEX]], 6
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[P]], i32 [[TMP5]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP6]], i32 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], splat (i64 1)
; CHECK-NEXT: call void @llvm.vp...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214476
More information about the llvm-commits
mailing list