[llvm] [LV] Convert gather loads with constant stride into strided loads (PR #147297)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 06:51:27 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/147297
>From b6f7a49cd740db5751fa0890439fc14d25373d53 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 30 Jun 2025 19:01:57 -0700
Subject: [PATCH 01/35] New VPWidenStridedLoadRecipe
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 51 +++++++++++++++-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 6 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 59 +++++++++++++++++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 ++--
5 files changed, 118 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8a770071752f3..2c830983e8c6d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3232,7 +3232,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
.Case([](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
.Case(
[](const VPWidenStoreRecipe *R) { return Instruction::Store; })
- .Case([](const VPWidenLoadRecipe *R) { return Instruction::Load; })
+ .Case<VPWidenLoadRecipe, VPWidenStridedLoadRecipe>(
+ [](const auto *R) { return Instruction::Load; })
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
[](const auto *R) { return Instruction::Call; })
.Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
@@ -3332,6 +3333,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPReductionPHISC:
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
+ case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4a5420185224b..1d23b13abac34 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -440,6 +440,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPWidenCastSC,
VPWidenGEPSC,
VPWidenIntrinsicSC,
+ VPWidenStridedLoadSC,
VPWidenLoadEVLSC,
VPWidenLoadSC,
VPWidenStoreEVLSC,
@@ -644,6 +645,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
@@ -3530,7 +3532,8 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
return R->getVPRecipeID() == VPRecipeBase::VPWidenLoadSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenStoreSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenLoadEVLSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC;
+ R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC ||
+ R->getVPRecipeID() == VPRecipeBase::VPWidenStridedLoadSC;
}
static inline bool classof(const VPUser *U) {
@@ -3651,6 +3654,52 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
#endif
};
+/// A recipe for strided load operations, using the base address, stride, and an
+/// optional mask. This recipe will generate an vp.strided.load intrinsic call
+/// to represent memory accesses with a fixed stride.
+struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
+ public VPRecipeValue {
+ VPWidenStridedLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Stride,
+ VPValue *VF, VPValue *Mask,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPWidenMemoryRecipe(
+ VPRecipeBase::VPWidenStridedLoadSC, Load, {Addr, Stride, VF},
+ /*Consecutive=*/false, /*Reverse=*/false, Metadata, DL),
+ VPRecipeValue(this, &Load) {
+ setMask(Mask);
+ }
+
+ VPWidenStridedLoadRecipe *clone() override {
+ return new VPWidenStridedLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
+ getStride(), getVF(), getMask(), *this,
+ getDebugLoc());
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStridedLoadSC);
+
+ /// Return the stride operand.
+ VPValue *getStride() const { return getOperand(1); }
+
+ /// Return the VF operand.
+ VPValue *getVF() const { return getOperand(2); }
+
+ /// Generate a strided load.
+ void execute(VPTransformState &State) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool usesFirstLaneOnly(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ return Op == getAddr() || Op == getStride() || Op == getVF();
+ }
+};
+
/// A recipe for widening store operations, using the stored value, the address
/// to store to and an optional mask.
struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 103dff1889a6a..7882187fa5314 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -211,8 +211,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) {
}
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) {
- assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)) &&
- "Store recipes should not define any values");
+ assert(
+ (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe, VPWidenStridedLoadRecipe>(
+ R)) &&
+ "Store recipes should not define any values");
return cast<LoadInst>(&R->getIngredient())->getType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2225dfa310c6c..f36c30bae0516 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -90,6 +90,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
+ case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
case VPWidenPHISC:
@@ -113,6 +114,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
case VPInstructionSC:
return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
+ case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
return true;
@@ -206,6 +208,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPInterleaveEVLSC:
case VPInterleaveSC:
return mayWriteToMemory();
+ case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
case VPWidenStoreEVLSC:
@@ -3700,9 +3703,11 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
- unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this)
- ? Instruction::Load
- : Instruction::Store;
+ unsigned Opcode =
+ isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe, VPWidenStridedLoadRecipe>(
+ this)
+ ? Instruction::Load
+ : Instruction::Store;
if (!Consecutive) {
// TODO: Using the original IR may not be accurate.
@@ -3721,8 +3726,13 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
assert(!IsReverseMask() &&
"Inconsecutive memory access should not have reverse order");
const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
- Type *PtrTy = Ptr->getType();
+ if (isa<VPWidenStridedLoadRecipe>(this))
+ return Ctx.TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
+ Ty, Ptr, IsMasked, Alignment, &Ingredient),
+ Ctx.CostKind);
+ Type *PtrTy = Ptr->getType();
// If the address value is uniform across all lanes, then the address can be
// calculated with scalar type and broadcast.
if (!vputils::isSingleScalar(getAddr()))
@@ -3850,6 +3860,47 @@ void VPWidenLoadEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPWidenStridedLoadRecipe::execute(VPTransformState &State) {
+ Type *ScalarDataTy = getLoadStoreType(&Ingredient);
+ auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+
+ auto &Builder = State.Builder;
+ Value *Addr = State.get(getAddr(), /*IsScalar*/ true);
+ Value *StrideInBytes = State.get(getStride(), /*IsScalar*/ true);
+ Value *Mask = nullptr;
+ if (VPValue *VPMask = getMask())
+ Mask = State.get(VPMask);
+ else
+ Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
+ Value *RunTimeVF = Builder.CreateZExtOrTrunc(State.get(getVF(), VPLane(0)),
+ Builder.getInt32Ty());
+
+ auto *PtrTy = Addr->getType();
+ auto *StrideTy = StrideInBytes->getType();
+ CallInst *NewLI = Builder.CreateIntrinsic(
+ Intrinsic::experimental_vp_strided_load, {DataTy, PtrTy, StrideTy},
+ {Addr, StrideInBytes, Mask, RunTimeVF}, nullptr, "wide.strided.load");
+ NewLI->addParamAttr(
+ 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
+ applyMetadata(*NewLI);
+ State.set(this, NewLI);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPWidenStridedLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN ";
+ printAsOperand(O, SlotTracker);
+ O << " = load ";
+ getAddr()->printAsOperand(O, SlotTracker);
+ O << ", stride = ";
+ getStride()->printAsOperand(O, SlotTracker);
+ O << ", runtimeVF = ";
+ getVF()->printAsOperand(O, SlotTracker);
+}
+#endif
+
void VPWidenStoreRecipe::execute(VPTransformState &State) {
VPValue *StoredVPValue = getStoredValue();
bool CreateScatter = !isConsecutive();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 98b468452dbc5..11a96ce7e2bb2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3057,10 +3057,12 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
- assert(all_of(Plan.getVF().users(),
- IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe>) &&
- "User of VF that we can't transform to EVL.");
+ assert(
+ all_of(
+ Plan.getVF().users(),
+ IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+ VPWidenIntOrFpInductionRecipe, VPWidenStridedLoadRecipe>) &&
+ "User of VF that we can't transform to EVL.");
Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
});
>From 986a90be0226153e058f79318edd65f7f343f161 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 7 Jul 2025 01:02:03 -0700
Subject: [PATCH 02/35] Expand VPVectorPointerRecipe to support stride
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 ++-
llvm/lib/Transforms/Vectorize/VPlan.h | 23 ++++++----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 8 +++-
.../VPlan/dissolve-replicate-regions.ll | 11 +++--
...ave-conditional-scalar-assignment-vplan.ll | 4 +-
.../LoopVectorize/VPlan/vplan-dot-printing.ll | 4 +-
.../VPlan/vplan-printing-before-execute.ll | 6 +--
.../LoopVectorize/VPlan/vplan-printing.ll | 46 +++++++++----------
.../widen-canonical-iv-register-pressure.ll | 12 ++---
9 files changed, 68 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2c830983e8c6d..a4eeb0df4ae5e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6429,8 +6429,11 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
Ptr, &Plan.getVF(), getLoadStoreType(I),
/*Stride*/ -1, ReverseFlags, VPI->getDebugLoc());
} else {
- VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I), Flags,
- VPI->getDebugLoc());
+ const DataLayout &DL = I->getDataLayout();
+ auto *StrideTy = DL.getIndexType(Ptr->getUnderlyingValue()->getType());
+ VPValue *StrideOne = Plan.getOrAddLiveIn(ConstantInt::get(StrideTy, 1));
+ VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I), StrideOne,
+ Flags, VPI->getDebugLoc());
}
Builder.setInsertPoint(VPI);
Builder.insert(VectorPtr);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1d23b13abac34..194f4937dcc4b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2216,21 +2216,25 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
};
/// A recipe to compute the pointers for widened memory accesses of \p
+/// SourceElementTy, with the \p Stride expressed in units of \p
/// SourceElementTy. Unrolling adds an extra offset operand for unrolled parts >
/// 0 and it produces `GEP Ptr, Offset`. The offset for unrolled part 0 is 0.
class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
public:
- VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
+ VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC,
+ ArrayRef<VPValue *>({Ptr, Stride}), GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
+ VPValue *getStride() const { return getOperand(1); }
+
VPValue *getOffset() {
- return getNumOperands() == 2 ? getOperand(1) : nullptr;
+ return getNumOperands() > 2 ? getOperand(2) : nullptr;
}
void execute(VPTransformState &State) override;
@@ -2252,8 +2256,9 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
}
VPVectorPointerRecipe *clone() override {
- auto *Clone = new VPVectorPointerRecipe(getOperand(0), SourceElementTy,
- getGEPNoWrapFlags(), getDebugLoc());
+ auto *Clone =
+ new VPVectorPointerRecipe(getOperand(0), SourceElementTy, getStride(),
+ getGEPNoWrapFlags(), getDebugLoc());
if (auto *Off = getOffset())
Clone->addOperand(Off);
return Clone;
@@ -3662,9 +3667,9 @@ struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
VPWidenStridedLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Stride,
VPValue *VF, VPValue *Mask,
const VPIRMetadata &Metadata, DebugLoc DL)
- : VPWidenMemoryRecipe(
- VPRecipeBase::VPWidenStridedLoadSC, Load, {Addr, Stride, VF},
- /*Consecutive=*/false, /*Reverse=*/false, Metadata, DL),
+ : VPWidenMemoryRecipe(VPRecipeBase::VPWidenStridedLoadSC, Load,
+ {Addr, Stride, VF},
+ /*Consecutive=*/false, Metadata, DL),
VPRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3689,7 +3694,7 @@ struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
+ VPSlotTracker &SlotTracker) const override;
#endif
/// Returns true if the recipe only uses the first lane of operand \p Op.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f1b9efae08377..4e3af417a1e98 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -352,8 +352,14 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
VPValue *VFxPart = Builder.createOverflowingOp(
Instruction::Mul, {VF, Plan.getConstantInt(IndexTy, Part)},
{true, true});
+ VPValue *Stride = Builder.createScalarZExtOrTrunc(
+ VPR->getStride(), IndexTy, TypeInfo.inferScalarType(VPR->getStride()),
+ DebugLoc::getUnknown());
+ VPValue *Offset =
+ Builder.createOverflowingOp(Instruction::Mul, {VFxPart, Stride});
Copy->setOperand(0, VPR->getOperand(0));
- Copy->addOperand(VFxPart);
+ Copy->setOperand(1, VPR->getOperand(1));
+ Copy->addOperand(Offset);
continue;
}
if (auto *Red = dyn_cast<VPReductionRecipe>(&R)) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index 31041a8cc5541..ced2cebb11e04 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -159,11 +159,12 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%16>/vp<[[VP5]]>
; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%25>/vp<[[VP5]]>
; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP7]]>
-; VECTOR-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer ir<%gep.dst>
-; VECTOR-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer ir<%gep.dst>, vp<[[VP13]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP14]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP15]]>, ir<%pred.val>.1
+; VECTOR-NEXT: EMIT vp<[[VP26:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: EMIT vp<[[VP27:%[0-9]+]]> = mul vp<[[VP26]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP28:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP29:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>, vp<[[VP27]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP28]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP29]]>, ir<%pred.val>.1
; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
index b4266483f2aa3..29f3276312551 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
@@ -37,7 +37,7 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP8:%[0-9]+]]>, vector.body ]
; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
; IC2-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
-; IC2-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<4>
+; IC2-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<1>, ir<4>
; IC2-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>
; IC2-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP6]]>
; IC2-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP3]]>, ir<%ld>
@@ -106,7 +106,7 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP2]]>
; IC2-TF-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP2]]>
; IC2-TF-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
-; IC2-TF-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<4>
+; IC2-TF-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<1>, ir<4>
; IC2-TF-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>, vp<[[VP8]]>
; IC2-TF-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP10]]>, vp<[[VP9]]>
; IC2-TF-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP3]]>, ir<%ld>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-dot-printing.ll
index ece1456b0ec19..aca6994c46f3f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-dot-printing.ll
@@ -40,11 +40,11 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) {
; CHECK-NEXT: "vector.body:\l" +
; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>, vp\<[[VF]]\>\l" +
; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer inbounds ir\<%arrayidx\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer inbounds ir\<%arrayidx\>, ir\<1\>\l" +
; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
; CHECK-NEXT: " WIDEN-INTRINSIC ir\<%call\> = call llvm.sqrt(ir\<%lv\>)\l" +
; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer inbounds ir\<%arrayidx2\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer inbounds ir\<%arrayidx2\>, ir\<1\>\l" +
; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT:%.+]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" +
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" +
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-before-execute.ll
index 26ed650b16f0f..832754daff41a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-before-execute.ll
@@ -28,10 +28,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; OPTIMIZE-NEXT: vector.body:
; OPTIMIZE-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; OPTIMIZE-NEXT: EMIT vp<%next.gep> = ptradd ir<%A>, vp<[[VP6]]>
-; OPTIMIZE-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; OPTIMIZE-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
; OPTIMIZE-NEXT: WIDEN ir<%l> = load vp<[[VP7]]>
; OPTIMIZE-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
-; OPTIMIZE-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; OPTIMIZE-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
; OPTIMIZE-NEXT: WIDEN store vp<[[VP8]]>, ir<%add>
; OPTIMIZE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; OPTIMIZE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -75,7 +75,7 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
-; FINAL-NEXT: vp<[[VP1:%[0-9]+]]> = vector-pointer ir<%A>, ir<8>
+; FINAL-NEXT: vp<[[VP1:%[0-9]+]]> = vector-pointer ir<%A>, ir<1>, ir<8>
; FINAL-NEXT: WIDEN ir<%l> = load ir<%A>
; FINAL-NEXT: WIDEN ir<%l>.1 = load vp<[[VP1]]>
; FINAL-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index d9eaab8d9a000..afd9a036cd706 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -26,11 +26,11 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN-INTRINSIC ir<%call> = call llvm.sqrt(ir<%lv>)
; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%call>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -54,7 +54,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) {
; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
; CHECK-NEXT: IR %lv = load float, ptr %arrayidx, align 4
-; CHECK-NEXT: IR %call = tail call float @llvm.sqrt.f32(float %lv)
+; CHECK-NEXT: IR %call = tail call float @llvm.sqrt.f32(float %lv) #2
; CHECK-NEXT: IR %arrayidx2 = getelementptr inbounds float, ptr %x, i64 %iv
; CHECK-NEXT: IR store float %call, ptr %arrayidx2, align 4
; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
@@ -102,13 +102,13 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z>
; CHECK-NEXT: WIDEN ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -206,7 +206,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: if.then.0:
; CHECK-NEXT: BLEND ir<%d> = ir<0> vp<%7>/ir<%cmp>
; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%idx>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%idx>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%d>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -386,7 +386,7 @@ define void @recipe_debug_loc_location(ptr nocapture %src) !dbg !5 {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%src>, vp<[[VP4]]>, !dbg /tmp/s.c:5:3
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%isd>, !dbg /tmp/s.c:6:3
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%isd>, ir<1>, !dbg /tmp/s.c:6:3
; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VP5]]>, !dbg /tmp/s.c:6:3
; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23>, !dbg /tmp/s.c:7:3
; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>, !dbg /tmp/s.c:8:3
@@ -413,7 +413,7 @@ define void @recipe_debug_loc_location(ptr nocapture %src) !dbg !5 {
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
; CHECK-NEXT: BLEND ir<%ysd.0> = ir<%psd> vp<%9>/vp<[[VP8]]>, !dbg /tmp/s.c:14:3
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%isd>, !dbg /tmp/s.c:15:3
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%isd>, ir<1>, !dbg /tmp/s.c:15:3
; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%ysd.0>, !dbg /tmp/s.c:15:3
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -572,7 +572,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -643,13 +643,13 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.y>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.y>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00>
; CHECK-NEXT: WIDEN ir<%mul> = fmul fast ir<%add>, ir<2.000000e+00>
; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00>
; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%div>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -723,12 +723,12 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20>
; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60>
; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -800,7 +800,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00>
; CHECK-NEXT: Successor(s): pred.call
@@ -826,7 +826,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[VP6]]>, vp<[[VP7]]>
; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value> ir<%fadd>/ir<%ifcond>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%st.addr>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%st.value>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -903,12 +903,12 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1>
; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3>
; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.x>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -980,7 +980,7 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%idx>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%idx>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l> to i64
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = extract-last-part ir<%zext>
@@ -1053,11 +1053,11 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.ptr>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.ptr>, ir<1>
; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-order splice ir<%for.1>, ir<%for.1.next>
; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP6]]>, ir<1>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep.ptr>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep.ptr>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%add>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1132,16 +1132,16 @@ define void @print_select_with_fastmath_flags(ptr noalias %a, ptr noalias %b, pt
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds nuw ir<%b>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%gep3> = getelementptr inbounds nuw ir<%c>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep3>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep3>, ir<1>
; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VP6]]>
; CHECK-NEXT: WIDEN ir<%cmp4> = fcmp ogt fast ir<%0>, ir<%1>
; CHECK-NEXT: WIDEN ir<%add> = fadd fast ir<%0>, ir<1.000000e+01>
; CHECK-NEXT: WIDEN ir<%cond> = select fast ir<%cmp4>, ir<%add>, ir<%1>
; CHECK-NEXT: CLONE ir<%gep11> = getelementptr inbounds nuw ir<%a>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep11>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%gep11>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%cond>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll b/llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll
index b823144f8e398..b4f2046bec09b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll
@@ -83,16 +83,16 @@ define i32 @two_reductions(i64 %N, ptr %a, ptr %b) {
; UF4-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; UF4-NEXT: CLONE ir<%ga> = getelementptr inbounds ir<%a>, vp<%index>
; UF4-NEXT: CLONE ir<%gb> = getelementptr inbounds ir<%b>, vp<%index>
-; UF4-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<4>
-; UF4-NEXT: vp<[[VP13:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<8>
-; UF4-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<12>
+; UF4-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<1>, ir<4>
+; UF4-NEXT: vp<[[VP13:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<1>, ir<8>
+; UF4-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer inbounds ir<%ga>, ir<1>, ir<12>
; UF4-NEXT: WIDEN ir<%la> = load ir<%ga>, vp<[[VP8]]>
; UF4-NEXT: WIDEN ir<%la>.1 = load vp<[[VP12]]>, vp<[[VP9]]>
; UF4-NEXT: WIDEN ir<%la>.2 = load vp<[[VP13]]>, vp<[[VP10]]>
; UF4-NEXT: WIDEN ir<%la>.3 = load vp<[[VP14]]>, vp<[[VP11]]>
-; UF4-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<4>
-; UF4-NEXT: vp<[[VP16:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<8>
-; UF4-NEXT: vp<[[VP17:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<12>
+; UF4-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<1>, ir<4>
+; UF4-NEXT: vp<[[VP16:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<1>, ir<8>
+; UF4-NEXT: vp<[[VP17:%[0-9]+]]> = vector-pointer inbounds ir<%gb>, ir<1>, ir<12>
; UF4-NEXT: WIDEN ir<%lb> = load ir<%gb>, vp<[[VP8]]>
; UF4-NEXT: WIDEN ir<%lb>.1 = load vp<[[VP15]]>, vp<[[VP9]]>
; UF4-NEXT: WIDEN ir<%lb>.2 = load vp<[[VP16]]>, vp<[[VP10]]>
>From 479e810e6384cfa644daaa43fafa30aaf7328bde Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 30 Jun 2025 20:38:38 -0700
Subject: [PATCH 03/35] Transform the gather to stride load
---
.../Transforms/Vectorize/LoopVectorize.cpp | 17 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 27 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 189 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 6 +
.../RISCV/masked_gather_scatter.ll | 44 +++-
.../runtime-check-dependent-on-stride.ll | 24 +--
.../LoopVectorize/RISCV/strided-accesses.ll | 142 ++++++++-----
.../RISCV/tail-folding-gather-scatter.ll | 43 +++-
.../RISCV/tail-folding-interleave.ll | 100 ++++-----
.../VPlan/vplan-print-after-all.ll | 1 +
10 files changed, 458 insertions(+), 135 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a4eeb0df4ae5e..0a112f691f28c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7091,6 +7091,17 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
Range);
}
+ // Interleave memory: for each Interleave Group we marked earlier as relevant
+ // for this VPlan, replace the Recipes widening its memory instructions with a
+ // single VPInterleaveRecipe at its insertion point.
+ RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
+ InterleaveGroups, CM.isEpilogueAllowed());
+
+ // Convert memory recipes to strided access recipes if the strided access is
+ // legal and profitable.
+ RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, CostCtx,
+ Range);
+
// Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
if (Range.Start.isScalar())
Range.End = Range.Start * 2;
@@ -7099,12 +7110,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
Plan->addVF(VF);
Plan->setName("Initial VPlan");
- // Interleave memory: for each Interleave Group we marked earlier as relevant
- // for this VPlan, replace the Recipes widening its memory instructions with a
- // single VPInterleaveRecipe at its insertion point.
- RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM.isEpilogueAllowed());
-
// Replace VPValues for known constant strides.
RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *Plan, PSE,
Legal->getLAI()->getSymbolicStrides());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 194f4937dcc4b..3ae3e41b677ba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2084,10 +2084,6 @@ class VPHistogramRecipe : public VPRecipeBase {
class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
- bool isPointerLoopInvariant() const {
- return getOperand(0)->isDefinedOutsideLoopRegions();
- }
-
bool isIndexLoopInvariant(unsigned I) const {
return getOperand(I + 1)->isDefinedOutsideLoopRegions();
}
@@ -2117,6 +2113,29 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
/// This recipe generates a GEP instruction.
unsigned getOpcode() const { return Instruction::GetElementPtr; }
+ bool isPointerLoopInvariant() const {
+ return getOperand(0)->isDefinedOutsideLoopRegions();
+ }
+
+ std::optional<unsigned> getUniqueVariantIndex() const {
+ std::optional<unsigned> VarIdx;
+ for (unsigned I = 0, E = getNumOperands() - 1; I < E; ++I) {
+ if (isIndexLoopInvariant(I))
+ continue;
+
+ if (VarIdx)
+ return std::nullopt;
+ VarIdx = I;
+ }
+ return VarIdx;
+ }
+
+ Type *getIndexedType(unsigned I) const {
+ auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
+ SmallVector<Value *, 4> Ops(GEP->idx_begin(), GEP->idx_begin() + I);
+ return GetElementPtrInst::getIndexedType(SourceElementTy, Ops);
+ }
+
/// Generate the gep nodes.
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 11a96ce7e2bb2..1db311cb663ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6508,3 +6508,192 @@ void VPlanTransforms::makeMemOpWideningDecisions(
ReplaceWith(Recipe);
}
}
+
+static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
+ // TODO: Support VPWidenPointerInductionRecipe.
+ if (auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(CurIndex))
+ return {WidenIV, WidenIV->getStepValue()};
+
+ auto *WidenR = dyn_cast<VPWidenRecipe>(CurIndex);
+ if (!WidenR || !CurIndex->getUnderlyingValue())
+ return {nullptr, nullptr};
+
+ unsigned Opcode = WidenR->getOpcode();
+ // TODO: Support Instruction::Add and Instruction::Or.
+ if (Opcode != Instruction::Shl && Opcode != Instruction::Mul)
+ return {nullptr, nullptr};
+
+ // Match the pattern binop(variant, invariant), or binop(invariant, variant)
+ // if the binary operator is commutative.
+ bool IsLHSUniform = vputils::isSingleScalar(WidenR->getOperand(0));
+ if (IsLHSUniform == vputils::isSingleScalar(WidenR->getOperand(1)) ||
+ (IsLHSUniform && !Instruction::isCommutative(Opcode)))
+ return {nullptr, nullptr};
+ unsigned VarIdx = IsLHSUniform ? 1 : 0;
+
+ auto [Start, Stride] = matchStridedStart(WidenR->getOperand(VarIdx));
+ if (!Start)
+ return {nullptr, nullptr};
+
+ SmallVector<VPValue *> StartOps(WidenR->operands());
+ StartOps[VarIdx] = Start;
+ auto *StartR = new VPReplicateRecipe(WidenR->getUnderlyingInstr(), StartOps,
+ /*IsUniform*/ true, /*Mask*/ nullptr,
+ /*Flags*/ *WidenR, /*Metadata*/ *WidenR,
+ WidenR->getDebugLoc());
+ StartR->insertBefore(WidenR);
+
+ unsigned InvIdx = VarIdx == 0 ? 1 : 0;
+ auto *StrideR =
+ new VPInstruction(Opcode, {Stride, WidenR->getOperand(InvIdx)},
+ VPRecipeWithIRFlags::WrapFlagsTy{false, false});
+ StrideR->insertBefore(WidenR);
+ return {StartR, StrideR};
+}
+
+static std::tuple<VPValue *, VPValue *, Type *>
+determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
+ // TODO: Check if the base pointer is strided.
+ if (!WidenGEP->isPointerLoopInvariant())
+ return {nullptr, nullptr, nullptr};
+
+ // Find the only one variant index.
+ std::optional<unsigned> VarIndex = WidenGEP->getUniqueVariantIndex();
+ if (!VarIndex)
+ return {nullptr, nullptr, nullptr};
+
+ Type *ElementTy = WidenGEP->getIndexedType(*VarIndex);
+ if (ElementTy->isScalableTy() || ElementTy->isStructTy() ||
+ ElementTy->isVectorTy())
+ return {nullptr, nullptr, nullptr};
+
+ unsigned VarOp = *VarIndex + 1;
+ VPValue *IndexVPV = WidenGEP->getOperand(VarOp);
+ auto [Start, Stride] = matchStridedStart(IndexVPV);
+ if (!Start)
+ return {nullptr, nullptr, nullptr};
+
+ SmallVector<VPValue *> Ops(WidenGEP->operands());
+ Ops[VarOp] = Start;
+ auto *BasePtr = new VPReplicateRecipe(
+ WidenGEP->getUnderlyingInstr(), Ops,
+ /*IsUniform*/ true, /*Mask*/ nullptr, /*Flags*/ *WidenGEP,
+ /*Metadata*/ {}, WidenGEP->getDebugLoc());
+ BasePtr->insertBefore(WidenGEP);
+
+ return {BasePtr, Stride, ElementTy};
+}
+
+void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range) {
+ if (Plan.hasScalarVFOnly())
+ return;
+
+ VPTypeAnalysis TypeInfo(Plan);
+ DenseMap<VPWidenGEPRecipe *, std::tuple<VPValue *, VPValue *, Type *>>
+ StrideCache;
+ SmallVector<VPRecipeBase *> ToErase;
+ SmallPtrSet<VPValue *, 4> PossiblyDead;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ auto *MemR = dyn_cast<VPWidenMemoryRecipe>(&R);
+ // TODO: Support strided store.
+ // TODO: Transform reverse access into strided access with -1 stride.
+ // TODO: Transform gather/scatter with uniform address into strided access
+ // with 0 stride.
+ // TODO: Transform interleave access into multiple strided accesses.
+ if (!MemR || !isa<VPWidenLoadRecipe>(MemR) || MemR->isConsecutive())
+ continue;
+
+ auto *Ptr = dyn_cast<VPWidenGEPRecipe>(MemR->getAddr());
+ if (!Ptr)
+ continue;
+
+ // Memory cost model requires the pointer operand of memory access
+ // instruction.
+ Value *PtrUV = Ptr->getUnderlyingValue();
+ if (!PtrUV)
+ continue;
+
+ // Try to get base and stride here.
+ VPValue *BasePtr, *StrideInElement;
+ Type *ElementTy;
+ auto It = StrideCache.find(Ptr);
+ if (It != StrideCache.end())
+ std::tie(BasePtr, StrideInElement, ElementTy) = It->second;
+ else
+ std::tie(BasePtr, StrideInElement, ElementTy) = StrideCache[Ptr] =
+ determineBaseAndStride(Ptr);
+
+ // Skip if the memory access is not a strided access.
+ if (!BasePtr) {
+ assert(!StrideInElement && !ElementTy);
+ continue;
+ }
+ assert(StrideInElement && ElementTy);
+
+ Instruction &Ingredient = MemR->getIngredient();
+ auto IsProfitable = [&](ElementCount VF) -> bool {
+ Type *DataTy = toVectorTy(getLoadStoreType(&Ingredient), VF);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment))
+ return false;
+ const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
+ const InstructionCost StridedLoadStoreCost =
+ Ctx.TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(
+ Intrinsic::experimental_vp_strided_load, DataTy, PtrUV,
+ MemR->isMasked(), Alignment, &Ingredient),
+ Ctx.CostKind);
+ return StridedLoadStoreCost < CurrentCost;
+ };
+
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
+ Range)) {
+ PossiblyDead.insert(BasePtr);
+ PossiblyDead.insert(StrideInElement);
+ continue;
+ }
+ PossiblyDead.insert(Ptr);
+
+ // Create a new vector pointer for strided access.
+ auto *GEP = dyn_cast<GetElementPtrInst>(PtrUV->stripPointerCasts());
+ auto *NewPtr = new VPVectorPointerRecipe(
+ BasePtr, ElementTy, StrideInElement,
+ GEP ? GEP->getNoWrapFlags() : GEPNoWrapFlags::none(),
+ Ptr->getDebugLoc());
+ NewPtr->insertBefore(MemR);
+
+ const DataLayout &DL = Ingredient.getDataLayout();
+ TypeSize TS = DL.getTypeAllocSize(ElementTy);
+ unsigned TypeScale = TS.getFixedValue();
+ VPValue *StrideInBytes = StrideInElement;
+ // Scale the stride by the size of the indexed type.
+ if (TypeScale != 1) {
+ VPValue *ScaleVPV = Plan.getOrAddLiveIn(ConstantInt::get(
+ TypeInfo.inferScalarType(StrideInElement), TypeScale));
+ auto *ScaledStride =
+ new VPInstruction(Instruction::Mul, {StrideInElement, ScaleVPV},
+ VPRecipeWithIRFlags::WrapFlagsTy{false, false});
+ ScaledStride->insertBefore(MemR);
+ StrideInBytes = ScaledStride;
+ }
+
+ auto *LoadR = cast<VPWidenLoadRecipe>(MemR);
+ auto *StridedLoad = new VPWidenStridedLoadRecipe(
+ *cast<LoadInst>(&Ingredient), NewPtr, StrideInBytes, &Plan.getVF(),
+ LoadR->getMask(), *LoadR, LoadR->getDebugLoc());
+ StridedLoad->insertBefore(LoadR);
+ LoadR->replaceAllUsesWith(StridedLoad);
+
+ ToErase.push_back(LoadR);
+ }
+ }
+
+ // Clean up dead memory access recipes, and unused base address and stride.
+ for (auto *R : ToErase)
+ R->eraseFromParent();
+ for (auto *V : PossiblyDead)
+ recursivelyDeleteDeadRecipes(V);
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6e11de399c406..b67b22d56f364 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -327,6 +327,12 @@ struct VPlanTransforms {
&InterleaveGroups,
const bool &EpilogueAllowed);
+ /// Transform widen memory recipes into strided access recipes when legal
+ /// and profitable. Clamps \p Range to maintain consistency with widen
+ /// decisions of \p Plan, and uses \p Ctx to evaluate the cost.
+ static void convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range);
+
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 2ca02df345060..f546afbfd11b9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -45,27 +45,39 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
; RV32-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; RV32: vector.ph:
+; RV32-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 1
; RV32-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; RV32-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP7]], splat (i64 16)
; RV32-NEXT: br label [[VECTOR_BODY:%.*]]
; RV32: vector.body:
+; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV32-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP10]], i64 0
+; RV32-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; RV32-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV32-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV32-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP13]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; RV32-NEXT: [[TMP25:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; RV32-NEXT: [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP25]], [[BROADCAST_SPLAT10]]
+; RV32-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 4
+; RV32-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP16]]
+; RV32-NEXT: [[TMP27:%.*]] = trunc i64 [[TMP13]] to i32
+; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP26]], i64 64, <vscale x 2 x i1> [[TMP15]], i32 [[TMP27]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; RV32-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV32-NEXT: [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
-; RV32-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP16]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]]
+; RV32-NEXT: [[TMP20:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP10]])
+; RV32-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP16]], 1
+; RV32-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP28]]
+; RV32-NEXT: [[TMP30:%.*]] = trunc i64 [[TMP13]] to i32
+; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP29]], i64 256, <vscale x 2 x i1> [[TMP20]], i32 [[TMP30]]), !alias.scope [[META5:![0-9]+]]
; RV32-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV32-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV32-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
; RV32-NEXT: call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> align 8 [[TMP19]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3]], !noalias [[META5]]
+; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; RV32-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; RV32-NEXT: [[TMP24:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -112,27 +124,39 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
; RV64-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; RV64: vector.ph:
+; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
; RV64-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; RV64-NEXT: [[TMP1:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP7]], splat (i64 16)
; RV64-NEXT: br label [[VECTOR_BODY:%.*]]
; RV64: vector.body:
+; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV64-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP10]], i64 0
+; RV64-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV64-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP4]], i64 0
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV64-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP13]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; RV64-NEXT: [[TMP20:%.*]] = icmp ult <vscale x 2 x i32> [[TMP12]], [[BROADCAST_SPLAT7]]
+; RV64-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 4
+; RV64-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP9]]
+; RV64-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP2]] to i32
+; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP25]], i64 64, <vscale x 2 x i1> [[TMP20]], i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV64-NEXT: [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
-; RV64-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP16]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP10]])
+; RV64-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP9]], 1
+; RV64-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP26]]
+; RV64-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP2]] to i32
+; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP15]], i64 256, <vscale x 2 x i1> [[TMP13]], i32 [[TMP16]]), !alias.scope [[META3:![0-9]+]]
; RV64-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV64-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV64-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
; RV64-NEXT: call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> align 8 [[TMP19]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; RV64-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; RV64-NEXT: [[TMP24:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
index a31bb408dcbc7..0e063897cd32c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
@@ -81,31 +81,31 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_PH]]:
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[STRIDE]], i64 0
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT15]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP35:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; NO-UNIT-STRIDE-MV-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 2 x i64> splat (i64 1), [[TMP35]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP27:%.*]] = shl nuw i64 [[TMP19]], 1
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP28:%.*]] = shl i64 [[STRIDE]], 3
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
-; NO-UNIT-STRIDE-MV-NEXT: [[STEP_ADD_3:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP26]], i64 0
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT14]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP25]], i64 0
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP33:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP24:%.*]] = icmp ult <vscale x 2 x i32> [[TMP33]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT16]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul i64 [[OFFSET_IDX]], [[STRIDE]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[OFFSET_IDX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP40:%.*]] = getelementptr i64, ptr [[P_STRIDED]], <vscale x 2 x i64> [[TMP34]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[P_STRIDED]], i64 [[TMP34]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_LOAD19:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP30]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META0:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP40]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP35:%.*]] = trunc i64 [[TMP27]] to i32
+; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP29]], i64 [[TMP28]], <vscale x 2 x i1> [[TMP24]], i32 [[TMP35]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
-; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; NO-UNIT-STRIDE-MV: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 0cff922ba09a3..9508f43498b80 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -11,20 +11,31 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 4 x i32> [[TMP5]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP14]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP9]], i64 32, <vscale x 4 x i1> [[TMP6]], i32 [[TMP13]])
; CHECK-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -57,12 +68,18 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-UF2-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; CHECK-UF2-NEXT: [[TMP9:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-UF2-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 4 x i64> [[STEP_ADD]], splat (i64 3)
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP8]]
; CHECK-UF2-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP9]]
; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 3
+; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP20]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP17]])
; CHECK-UF2-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-UF2-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], splat (i32 1)
; CHECK-UF2-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true))
@@ -113,21 +130,32 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP6]], splat (i64 64)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP11]], 6
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ult <vscale x 4 x i32> [[TMP14]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP15]], i64 256, <vscale x 4 x i1> [[TMP8]], i32 [[TMP16]])
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -161,10 +189,16 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[TMP6]]
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = shl i64 [[INDEX]], 6
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP17]]
; CHECK-UF2-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
; CHECK-UF2-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[STEP_ADD]]
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 6
+; CHECK-UF2-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
+; CHECK-UF2-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; CHECK-UF2-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-UF2-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], splat (i32 1)
; CHECK-UF2-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true))
@@ -779,23 +813,34 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; STRIDED-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; STRIDED: [[VECTOR_PH]]:
+; STRIDED-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP42]], 2
+; STRIDED-NEXT: [[TMP46:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED: [[VECTOR_BODY]]:
+; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP12]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP43:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; STRIDED-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP43]], i64 0
+; STRIDED-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT11]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-NEXT: [[TMP44:%.*]] = zext i32 [[TMP43]] to i64
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP44]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; STRIDED-NEXT: [[TMP47:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; STRIDED-NEXT: [[TMP48:%.*]] = icmp ult <vscale x 4 x i32> [[TMP47]], [[BROADCAST_SPLAT12]]
+; STRIDED-NEXT: [[TMP49:%.*]] = mul nuw nsw i64 [[INDEX]], [[STRIDE]]
; STRIDED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT1]]
-; STRIDED-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP18]]
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META5:![0-9]+]]
+; STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP49]]
+; STRIDED-NEXT: [[TMP51:%.*]] = trunc i64 [[TMP45]] to i32
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP50]], i64 [[TMP46]], <vscale x 4 x i1> [[TMP48]], i32 [[TMP51]]), !alias.scope [[META5:![0-9]+]]
; STRIDED-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP18]]
; STRIDED-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META8:![0-9]+]], !noalias [[META5]]
+; STRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP44]], [[INDEX]]
; STRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP44]]
; STRIDED-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; STRIDED-NEXT: [[TMP41:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -881,6 +926,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 1
; STRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP30]]
; STRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; STRIDED-UF2-NEXT: [[TMP32:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT10]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-UF2-NEXT: [[TMP31:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -889,12 +935,16 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP31]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; STRIDED-UF2-NEXT: [[TMP35:%.*]] = mul nuw nsw i64 [[INDEX]], [[STRIDE]]
; STRIDED-UF2-NEXT: [[TMP33:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT11]]
; STRIDED-UF2-NEXT: [[TMP34:%.*]] = mul nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT11]]
-; STRIDED-UF2-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP33]]
-; STRIDED-UF2-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP34]]
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP35]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META8:![0-9]+]]
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP36]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META8]]
+; STRIDED-UF2-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP35]]
+; STRIDED-UF2-NEXT: [[TMP42:%.*]] = mul i64 [[TMP29]], [[STRIDE]]
+; STRIDED-UF2-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP36]], i64 [[TMP42]]
+; STRIDED-UF2-NEXT: [[TMP44:%.*]] = trunc i64 [[TMP29]] to i32
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP36]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP44]]), !alias.scope [[META8:![0-9]+]]
+; STRIDED-UF2-NEXT: [[TMP45:%.*]] = trunc i64 [[TMP29]] to i32
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP43]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP45]]), !alias.scope [[META8]]
; STRIDED-UF2-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER12]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP33]]
@@ -1304,23 +1354,25 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
; NOSTRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; NOSTRIDED: [[VECTOR_PH]]:
-; NOSTRIDED-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NOSTRIDED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
; NOSTRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NOSTRIDED-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NOSTRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; NOSTRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; NOSTRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; NOSTRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[VEC_IND]]
-; NOSTRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; NOSTRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP2]], i64 0
+; NOSTRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; NOSTRIDED-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; NOSTRIDED-NEXT: [[TMP4:%.*]] = icmp ult <vscale x 2 x i32> [[TMP8]], [[BROADCAST_SPLAT]]
+; NOSTRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
+; NOSTRIDED-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
+; NOSTRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> [[TMP4]], i32 [[TMP6]])
; NOSTRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; NOSTRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
; NOSTRIDED-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP3]], [[EVL_BASED_IV]]
; NOSTRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
-; NOSTRIDED-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NOSTRIDED-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; NOSTRIDED: [[MIDDLE_BLOCK]]:
@@ -1338,27 +1390,23 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-UF2: [[VECTOR_PH]]:
; NOSTRIDED-UF2-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; NOSTRIDED-UF2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; NOSTRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; NOSTRIDED-UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; NOSTRIDED-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; NOSTRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; NOSTRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; NOSTRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED-UF2: [[VECTOR_BODY]]:
; NOSTRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NOSTRIDED-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NOSTRIDED-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; NOSTRIDED-UF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[VEC_IND]]
-; NOSTRIDED-UF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[STEP_ADD]]
-; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
-; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
+; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
+; NOSTRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; NOSTRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
+; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; NOSTRIDED-UF2-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
+; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
; NOSTRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
; NOSTRIDED-UF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i64 [[TMP3]]
; NOSTRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP9]], align 8
; NOSTRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], ptr [[TMP12]], align 8
; NOSTRIDED-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; NOSTRIDED-UF2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; NOSTRIDED-UF2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NOSTRIDED-UF2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; NOSTRIDED-UF2: [[MIDDLE_BLOCK]]:
@@ -1384,23 +1432,25 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-NEXT: [[ENTRY:.*:]]
; STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; STRIDED: [[VECTOR_PH]]:
-; STRIDED-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; STRIDED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED: [[VECTOR_BODY]]:
; STRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; STRIDED-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; STRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[VEC_IND]]
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP2]], i64 0
+; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; STRIDED-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; STRIDED-NEXT: [[TMP4:%.*]] = icmp ult <vscale x 2 x i32> [[TMP8]], [[BROADCAST_SPLAT]]
+; STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
+; STRIDED-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> [[TMP4]], i32 [[TMP6]])
; STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; STRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; STRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
; STRIDED-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP3]], [[EVL_BASED_IV]]
; STRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
-; STRIDED-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; STRIDED-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; STRIDED-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; STRIDED: [[MIDDLE_BLOCK]]:
@@ -1418,27 +1468,23 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-UF2: [[VECTOR_PH]]:
; STRIDED-UF2-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; STRIDED-UF2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; STRIDED-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; STRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; STRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; STRIDED-UF2-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; STRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED-UF2: [[VECTOR_BODY]]:
; STRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; STRIDED-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; STRIDED-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; STRIDED-UF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[VEC_IND]]
-; STRIDED-UF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], <vscale x 2 x i64> [[STEP_ADD]]
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
+; STRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
+; STRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; STRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; STRIDED-UF2-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
; STRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
; STRIDED-UF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i64 [[TMP3]]
; STRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP9]], align 8
; STRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], ptr [[TMP12]], align 8
; STRIDED-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; STRIDED-UF2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; STRIDED-UF2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; STRIDED-UF2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; STRIDED-UF2: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
index a0fca8fd78a21..c5c6a2597046c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
@@ -27,18 +27,45 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
;
; NO-VP-LABEL: @gather_scatter(
; NO-VP-NEXT: entry:
+; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP9]]
+; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; NO-VP: vector.ph:
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label [[FOR_BODY1:%.*]]
-; NO-VP: for.body:
-; NO-VP-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; NO-VP: vector.body:
+; NO-VP-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV1]]
-; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
-; NO-VP-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], i64 [[TMP0]]
+; NO-VP-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
+; NO-VP-NEXT: [[WIDE_STRIDED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[ARRAYIDX3]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP6]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x float> poison)
+; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
+; NO-VP-NEXT: call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> [[WIDE_MASKED_GATHER]], <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true))
+; NO-VP-NEXT: [[INDVARS_IV_NEXT1]] = add nuw i64 [[INDVARS_IV1]], [[TMP3]]
+; NO-VP-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT1]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; NO-VP: middle.block:
+; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; NO-VP: scalar.ph:
+; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; NO-VP-NEXT: br label [[FOR_BODY:%.*]]
+; NO-VP: for.body:
+; NO-VP-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[INDEX]], i64 [[INDVARS_IV]]
+; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX4]], align 8
+; NO-VP-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN]], i64 [[TMP0]]
; NO-VP-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-; NO-VP-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]
+; NO-VP-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP0]]
; NO-VP-NEXT: store float [[TMP1]], ptr [[ARRAYIDX7]], align 4
-; NO-VP-NEXT: [[INDVARS_IV_NEXT1]] = add nuw nsw i64 [[INDVARS_IV1]], 1
-; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT1]], [[N:%.*]]
-; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY1]]
+; NO-VP-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; NO-VP: for.end:
; NO-VP-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 2091cddc10bde..6dabdfc5edc3d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -344,28 +344,34 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
-; IF-EVL-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
-; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP4]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT: [[TMP7:%.*]] = icmp ult <vscale x 4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0
+; IF-EVL-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 1
+; IF-EVL-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP1]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP16]])
; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
-; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 2
+; IF-EVL-NEXT: [[TMP18:%.*]] = trunc i64 [[TMP1]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP18]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
+; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
-; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; IF-EVL: middle.block:
@@ -559,35 +565,39 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
-; IF-EVL-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[INDUCTION:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP4]]
+; IF-EVL-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
-; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT: [[TMP8:%.*]] = sub nsw i64 0, [[TMP7]]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT: [[TMP20:%.*]] = icmp ult <vscale x 4 x i32> [[TMP5]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT: [[TMP21:%.*]] = sub i64 [[N]], [[INDEX]]
+; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP21]], i32 0
+; IF-EVL-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP3]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP9]])
; IF-EVL-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 1
+; IF-EVL-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP3]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]
-; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 2
+; IF-EVL-NEXT: [[TMP23:%.*]] = trunc i64 [[TMP3]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP23]])
; IF-EVL-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
-; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 3
+; IF-EVL-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP3]] to i32
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP24]])
; IF-EVL-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]
; IF-EVL-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; IF-EVL: middle.block:
@@ -611,32 +621,28 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]]
-; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
-; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; NO-VP-NEXT: [[INDUCTION:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
-; NO-VP-NEXT: [[TMP9:%.*]] = sub nsw i64 0, [[TMP5]]
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
-; NO-VP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[INDEX]]
+; NO-VP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP7]], i32 0
+; NO-VP-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; NO-VP-NEXT: [[TMP23:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP23]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 1
+; NO-VP-NEXT: [[TMP23:%.*]] = trunc i64 [[TMP5]] to i32
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP23]])
; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]
-; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 2
+; NO-VP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP5]] to i32
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
; NO-VP-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]
-; NO-VP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 3
+; NO-VP-NEXT: [[TMP26:%.*]] = trunc i64 [[TMP5]] to i32
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP26]])
; NO-VP-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; NO-VP-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; NO-VP: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 4bc9a8d96e542..9b77d697fcf46 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -24,6 +24,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createPartialReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInterleaveGroups
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::convertToStridedAccesses
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::dropPoisonGeneratingRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::hoistPredicatedLoads
>From 7a8dab3550cf466c53080107b0443afbbea51a53 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 22 Aug 2025 01:28:33 -0700
Subject: [PATCH 04/35] Support EVL
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++
.../RISCV/masked_gather_scatter.ll | 26 ++---------
.../runtime-check-dependent-on-stride.ll | 9 +---
.../LoopVectorize/RISCV/strided-accesses.ll | 45 +++----------------
.../RISCV/tail-folding-gather-scatter.ll | 28 +++++++-----
.../RISCV/tail-folding-interleave.ll | 33 +++-----------
6 files changed, 43 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1db311cb663ae..3a46a0ed0889e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2935,6 +2935,14 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
TypeInfo.inferScalarType(LoadR), {}, {}, DL);
}
+ if (auto *StridedL = dyn_cast<VPWidenStridedLoadRecipe>(&CurRecipe))
+ if (StridedL->isMasked() &&
+ match(StridedL->getMask(), m_RemoveMask(HeaderMask, Mask)))
+ return new VPWidenStridedLoadRecipe(
+ *cast<LoadInst>(&StridedL->getIngredient()), StridedL->getAddr(),
+ StridedL->getStride(), &EVL, Mask, *StridedL,
+ StridedL->getDebugLoc());
+
VPValue *StoredVal;
if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal),
m_RemoveMask(HeaderMask, Mask))))
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index f546afbfd11b9..82c85e6a5a801 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -45,8 +45,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
; RV32-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; RV32: vector.ph:
-; RV32-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 1
; RV32-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; RV32-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP7]], splat (i64 16)
; RV32-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -55,24 +53,17 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; RV32-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP10]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; RV32-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV32-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV32-NEXT: [[TMP25:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; RV32-NEXT: [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP25]], [[BROADCAST_SPLAT10]]
; RV32-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 4
; RV32-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP16]]
-; RV32-NEXT: [[TMP27:%.*]] = trunc i64 [[TMP13]] to i32
-; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP26]], i64 64, <vscale x 2 x i1> [[TMP15]], i32 [[TMP27]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP26]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; RV32-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV32-NEXT: [[TMP20:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP10]])
; RV32-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP16]], 1
; RV32-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP28]]
-; RV32-NEXT: [[TMP30:%.*]] = trunc i64 [[TMP13]] to i32
-; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP29]], i64 256, <vscale x 2 x i1> [[TMP20]], i32 [[TMP30]]), !alias.scope [[META5:![0-9]+]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP29]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]]
; RV32-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV32-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV32-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
@@ -124,8 +115,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
; RV64-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; RV64: vector.ph:
-; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
; RV64-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; RV64-NEXT: [[TMP1:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP7]], splat (i64 16)
; RV64-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -134,24 +123,17 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; RV64-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP10]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV64-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP4]], i64 0
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV64-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; RV64-NEXT: [[TMP20:%.*]] = icmp ult <vscale x 2 x i32> [[TMP12]], [[BROADCAST_SPLAT7]]
; RV64-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 4
; RV64-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP9]]
-; RV64-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP2]] to i32
-; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP25]], i64 64, <vscale x 2 x i1> [[TMP20]], i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP25]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV64-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP10]])
; RV64-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP9]], 1
; RV64-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP26]]
-; RV64-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP2]] to i32
-; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP15]], i64 256, <vscale x 2 x i1> [[TMP13]], i32 [[TMP16]]), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP15]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
; RV64-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV64-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV64-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
index 0e063897cd32c..8a6108a0248ba 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
@@ -81,26 +81,19 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_PH]]:
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP27:%.*]] = shl nuw i64 [[TMP19]], 1
; NO-UNIT-STRIDE-MV-NEXT: [[TMP28:%.*]] = shl i64 [[STRIDE]], 3
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP25]], i64 0
-; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP33:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP24:%.*]] = icmp ult <vscale x 2 x i32> [[TMP33]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul i64 [[OFFSET_IDX]], [[STRIDE]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[P_STRIDED]], i64 [[TMP34]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_LOAD19:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP30]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META0:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP35:%.*]] = trunc i64 [[TMP27]] to i32
-; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP29]], i64 [[TMP28]], <vscale x 2 x i1> [[TMP24]], i32 [[TMP35]]), !alias.scope [[META3:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP29]], i64 [[TMP28]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 9508f43498b80..a6bada6bc6332 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -11,8 +11,6 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -20,19 +18,14 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 4 x i32> [[TMP5]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP14]]
-; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP9]], i64 32, <vscale x 4 x i1> [[TMP6]], i32 [[TMP13]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP9]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
@@ -130,8 +123,6 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP6]], splat (i64 64)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -140,19 +131,14 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP11]], 6
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ult <vscale x 4 x i32> [[TMP14]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP15]], i64 256, <vscale x 4 x i1> [[TMP8]], i32 [[TMP16]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP15]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
@@ -813,8 +799,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; STRIDED-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; STRIDED: [[VECTOR_PH]]:
-; STRIDED-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP42]], 2
; STRIDED-NEXT: [[TMP46:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -825,18 +809,13 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP12]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP43:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; STRIDED-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP43]], i64 0
-; STRIDED-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT11]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-NEXT: [[TMP44:%.*]] = zext i32 [[TMP43]] to i64
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP44]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; STRIDED-NEXT: [[TMP47:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; STRIDED-NEXT: [[TMP48:%.*]] = icmp ult <vscale x 4 x i32> [[TMP47]], [[BROADCAST_SPLAT12]]
; STRIDED-NEXT: [[TMP49:%.*]] = mul nuw nsw i64 [[INDEX]], [[STRIDE]]
; STRIDED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT1]]
; STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP49]]
-; STRIDED-NEXT: [[TMP51:%.*]] = trunc i64 [[TMP45]] to i32
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP50]], i64 [[TMP46]], <vscale x 4 x i1> [[TMP48]], i32 [[TMP51]]), !alias.scope [[META5:![0-9]+]]
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP50]], i64 [[TMP46]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META5:![0-9]+]]
; STRIDED-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP18]]
; STRIDED-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META8:![0-9]+]], !noalias [[META5]]
@@ -1354,20 +1333,13 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
; NOSTRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; NOSTRIDED: [[VECTOR_PH]]:
-; NOSTRIDED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
; NOSTRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NOSTRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP2]], i64 0
-; NOSTRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; NOSTRIDED-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; NOSTRIDED-NEXT: [[TMP4:%.*]] = icmp ult <vscale x 2 x i32> [[TMP8]], [[BROADCAST_SPLAT]]
; NOSTRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
-; NOSTRIDED-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
-; NOSTRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> [[TMP4]], i32 [[TMP6]])
+; NOSTRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; NOSTRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; NOSTRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
@@ -1432,20 +1404,13 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-NEXT: [[ENTRY:.*:]]
; STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; STRIDED: [[VECTOR_PH]]:
-; STRIDED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED: [[VECTOR_BODY]]:
; STRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP2]], i64 0
-; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; STRIDED-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; STRIDED-NEXT: [[TMP4:%.*]] = icmp ult <vscale x 2 x i32> [[TMP8]], [[BROADCAST_SPLAT]]
; STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
-; STRIDED-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> [[TMP4]], i32 [[TMP6]])
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; STRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; STRIDED-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
index c5c6a2597046c..be4300f2c3619 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
@@ -11,17 +11,25 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
; IF-EVL-LABEL: @gather_scatter(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY1:%.*]]
-; IF-EVL: for.body:
-; IF-EVL-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[SCALAR_PH:%.*]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
+; IF-EVL: vector.body:
+; IF-EVL-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[FOR_BODY1]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[FOR_BODY1]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV1]]
-; IF-EVL-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
-; IF-EVL-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], i64 [[TMP0]]
-; IF-EVL-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-; IF-EVL-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]
-; IF-EVL-NEXT: store float [[TMP1]], ptr [[ARRAYIDX7]], align 4
-; IF-EVL-NEXT: [[INDVARS_IV_NEXT1]] = add nuw nsw i64 [[INDVARS_IV1]], 1
-; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT1]], [[N:%.*]]
-; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY1]]
+; IF-EVL-NEXT: [[WIDE_STRIDED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[ARRAYIDX3]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x float> @llvm.vp.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP2]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
+; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> [[WIDE_MASKED_GATHER]], <vscale x 2 x ptr> align 4 [[TMP3]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDVARS_IV1]]
+; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; IF-EVL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; IF-EVL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IF-EVL: middle.block:
+; IF-EVL-NEXT: br label [[FOR_END:%.*]]
; IF-EVL: for.end:
; IF-EVL-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 6dabdfc5edc3d..7236d0265216c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -344,29 +344,20 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
-; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP4]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; IF-EVL-NEXT: [[TMP7:%.*]] = icmp ult <vscale x 4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0
-; IF-EVL-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP6]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 1
-; IF-EVL-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP1]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP16]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 2
-; IF-EVL-NEXT: [[TMP18:%.*]] = trunc i64 [[TMP1]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 4 x i1> [[TMP7]], i32 [[TMP18]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
@@ -565,34 +556,24 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
-; IF-EVL-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; IF-EVL-NEXT: [[TMP20:%.*]] = icmp ult <vscale x 4 x i32> [[TMP5]], [[BROADCAST_SPLAT]]
; IF-EVL-NEXT: [[TMP21:%.*]] = sub i64 [[N]], [[INDEX]]
; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP21]], i32 0
-; IF-EVL-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP3]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP9]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 1
-; IF-EVL-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP3]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP12]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 2
-; IF-EVL-NEXT: [[TMP23:%.*]] = trunc i64 [[TMP3]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP23]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 3
-; IF-EVL-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP3]] to i32
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, <vscale x 4 x i1> [[TMP20]], i32 [[TMP24]])
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]
; IF-EVL-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
>From 78cde7c1e3d889cb679e48002712c5b539d40fae Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 21 Oct 2025 23:23:05 -0700
Subject: [PATCH 05/35] Update comment. nfc
---
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +++++---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++++--
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3ae3e41b677ba..9bd630c61f20d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2130,6 +2130,7 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
return VarIdx;
}
+ /// Returns the element type for the first \p I indices of this recipe.
Type *getIndexedType(unsigned I) const {
auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
SmallVector<Value *, 4> Ops(GEP->idx_begin(), GEP->idx_begin() + I);
@@ -3678,9 +3679,9 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
#endif
};
-/// A recipe for strided load operations, using the base address, stride, and an
-/// optional mask. This recipe will generate an vp.strided.load intrinsic call
-/// to represent memory accesses with a fixed stride.
+/// A recipe for strided load operations, using the base address, stride, VF,
+/// and an optional mask. This recipe will generate a vp.strided.load intrinsic
+/// call to represent memory accesses with a fixed stride.
struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
public VPRecipeValue {
VPWidenStridedLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Stride,
@@ -3720,6 +3721,7 @@ struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
bool usesFirstLaneOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
+ // All operands except the mask are only used for the first lane.
return Op == getAddr() || Op == getStride() || Op == getVF();
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3a46a0ed0889e..d29dd72a8f3d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6531,8 +6531,8 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
if (Opcode != Instruction::Shl && Opcode != Instruction::Mul)
return {nullptr, nullptr};
- // Match the pattern binop(variant, invariant), or binop(invariant, variant)
- // if the binary operator is commutative.
+ // Match the pattern binop(variant, uniform), or binop(uniform, variant) if
+ // the binary operator is commutative.
bool IsLHSUniform = vputils::isSingleScalar(WidenR->getOperand(0));
if (IsLHSUniform == vputils::isSingleScalar(WidenR->getOperand(1)) ||
(IsLHSUniform && !Instruction::isCommutative(Opcode)))
@@ -6559,6 +6559,10 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
return {StartR, StrideR};
}
+/// Checks if the given VPWidenGEPRecipe \p WidenGEP represents a strided
+/// access. If so, it creates recipes representing the base pointer and stride
+/// in element type, and returns a tuple of {base pointer, stride, element
+/// type}. Otherwise, returns a tuple where all elements are nullptr.
static std::tuple<VPValue *, VPValue *, Type *>
determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
// TODO: Check if the base pointer is strided.
>From eca0fe9137081b007169cd1ab4511deb6e1227c9 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 22 Oct 2025 02:15:56 -0700
Subject: [PATCH 06/35] Find base pointer and stride after profitable check
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 50 +++++++++----------
1 file changed, 23 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d29dd72a8f3d8..8a24afe3269e8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6604,8 +6604,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
VPTypeAnalysis TypeInfo(Plan);
DenseMap<VPWidenGEPRecipe *, std::tuple<VPValue *, VPValue *, Type *>>
StrideCache;
- SmallVector<VPRecipeBase *> ToErase;
- SmallPtrSet<VPValue *, 4> PossiblyDead;
+ SmallVector<VPWidenMemoryRecipe *> ToErase;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -6628,23 +6627,6 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
if (!PtrUV)
continue;
- // Try to get base and stride here.
- VPValue *BasePtr, *StrideInElement;
- Type *ElementTy;
- auto It = StrideCache.find(Ptr);
- if (It != StrideCache.end())
- std::tie(BasePtr, StrideInElement, ElementTy) = It->second;
- else
- std::tie(BasePtr, StrideInElement, ElementTy) = StrideCache[Ptr] =
- determineBaseAndStride(Ptr);
-
- // Skip if the memory access is not a strided access.
- if (!BasePtr) {
- assert(!StrideInElement && !ElementTy);
- continue;
- }
- assert(StrideInElement && ElementTy);
-
Instruction &Ingredient = MemR->getIngredient();
auto IsProfitable = [&](ElementCount VF) -> bool {
Type *DataTy = toVectorTy(getLoadStoreType(&Ingredient), VF);
@@ -6662,12 +6644,25 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
};
if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
- Range)) {
- PossiblyDead.insert(BasePtr);
- PossiblyDead.insert(StrideInElement);
+ Range))
+ continue;
+
+ // Try to get base and stride here.
+ VPValue *BasePtr, *StrideInElement;
+ Type *ElementTy;
+ auto It = StrideCache.find(Ptr);
+ if (It != StrideCache.end())
+ std::tie(BasePtr, StrideInElement, ElementTy) = It->second;
+ else
+ std::tie(BasePtr, StrideInElement, ElementTy) = StrideCache[Ptr] =
+ determineBaseAndStride(Ptr);
+
+ // Skip if the memory access is not a strided access.
+ if (!BasePtr) {
+ assert(!StrideInElement && !ElementTy);
continue;
}
- PossiblyDead.insert(Ptr);
+ assert(StrideInElement && ElementTy);
// Create a new vector pointer for strided access.
auto *GEP = dyn_cast<GetElementPtrInst>(PtrUV->stripPointerCasts());
@@ -6703,9 +6698,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
}
}
- // Clean up dead memory access recipes, and unused base address and stride.
- for (auto *R : ToErase)
+ // Clean up dead recipes.
+ for (auto *R : ToErase) {
+ VPValue *Addr = R->getAddr();
R->eraseFromParent();
- for (auto *V : PossiblyDead)
- recursivelyDeleteDeadRecipes(V);
+ recursivelyDeleteDeadRecipes(Addr);
+ }
}
>From a9bd13f83642a3a509da08313d9e79f90556e78a Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 22 Oct 2025 02:43:30 -0700
Subject: [PATCH 07/35] Replace getUnderlyingValue() with getUnderlyingInstr()
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8a24afe3269e8..6bf734c079190 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6523,7 +6523,7 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
return {WidenIV, WidenIV->getStepValue()};
auto *WidenR = dyn_cast<VPWidenRecipe>(CurIndex);
- if (!WidenR || !CurIndex->getUnderlyingValue())
+ if (!WidenR || !WidenR->getUnderlyingInstr())
return {nullptr, nullptr};
unsigned Opcode = WidenR->getOpcode();
>From a1029f316a2897780f10e63c447452daa90cfe35 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 22 Oct 2025 03:11:24 -0700
Subject: [PATCH 08/35] Directly use flags from WidenGEP
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6bf734c079190..da2e6576165fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6665,10 +6665,8 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
assert(StrideInElement && ElementTy);
// Create a new vector pointer for strided access.
- auto *GEP = dyn_cast<GetElementPtrInst>(PtrUV->stripPointerCasts());
auto *NewPtr = new VPVectorPointerRecipe(
- BasePtr, ElementTy, StrideInElement,
- GEP ? GEP->getNoWrapFlags() : GEPNoWrapFlags::none(),
+ BasePtr, ElementTy, StrideInElement, Ptr->getGEPNoWrapFlags(),
Ptr->getDebugLoc());
NewPtr->insertBefore(MemR);
>From bdb1e5925f9dec97b0dd82bb1958cb26f3312f2c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 22 Oct 2025 03:29:02 -0700
Subject: [PATCH 09/35] Pass Ptr->getUnderlyingValue() in memory cost model
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index da2e6576165fe..311be16b0dbc6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6621,12 +6621,6 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
if (!Ptr)
continue;
- // Memory cost model requires the pointer operand of memory access
- // instruction.
- Value *PtrUV = Ptr->getUnderlyingValue();
- if (!PtrUV)
- continue;
-
Instruction &Ingredient = MemR->getIngredient();
auto IsProfitable = [&](ElementCount VF) -> bool {
Type *DataTy = toVectorTy(getLoadStoreType(&Ingredient), VF);
@@ -6637,8 +6631,9 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
const InstructionCost StridedLoadStoreCost =
Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
- Intrinsic::experimental_vp_strided_load, DataTy, PtrUV,
- MemR->isMasked(), Alignment, &Ingredient),
+ Intrinsic::experimental_vp_strided_load, DataTy,
+ Ptr->getUnderlyingValue(), MemR->isMasked(), Alignment,
+ &Ingredient),
Ctx.CostKind);
return StridedLoadStoreCost < CurrentCost;
};
>From ae171476e409d3780bd8ca4a50171e49ca4b92bb Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 22 Oct 2025 06:30:47 -0700
Subject: [PATCH 10/35] Implement VPWidenStridedLoadRecipe::computeCost
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 27 +++++++++++--------
2 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 9bd630c61f20d..72bfba7935a69 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3711,6 +3711,10 @@ struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
/// Generate a strided load.
void execute(VPTransformState &State) override;
+ /// Return the cost of this VPWidenStridedLoadRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f36c30bae0516..d8d1d53fe40e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3703,11 +3703,9 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
- unsigned Opcode =
- isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe, VPWidenStridedLoadRecipe>(
- this)
- ? Instruction::Load
- : Instruction::Store;
+ unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this)
+ ? Instruction::Load
+ : Instruction::Store;
if (!Consecutive) {
// TODO: Using the original IR may not be accurate.
@@ -3726,13 +3724,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
assert(!IsReverseMask() &&
"Inconsecutive memory access should not have reverse order");
const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
- if (isa<VPWidenStridedLoadRecipe>(this))
- return Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
- Ty, Ptr, IsMasked, Alignment, &Ingredient),
- Ctx.CostKind);
-
Type *PtrTy = Ptr->getType();
+
// If the address value is uniform across all lanes, then the address can be
// calculated with scalar type and broadcast.
if (!vputils::isSingleScalar(getAddr()))
@@ -3887,6 +3880,18 @@ void VPWidenStridedLoadRecipe::execute(VPTransformState &State) {
State.set(this, NewLI);
}
+InstructionCost
+VPWidenStridedLoadRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
+ const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ return Ctx.TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load, Ty,
+ Ptr, IsMasked, Alignment, &Ingredient),
+ Ctx.CostKind);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenStridedLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
>From 73d18ef9951f705aba99059b91eee069d5769e25 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 7 Nov 2025 00:48:22 -0800
Subject: [PATCH 11/35] update comments. nfc
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 311be16b0dbc6..4b25eaf396203 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6561,7 +6561,7 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
/// Checks if the given VPWidenGEPRecipe \p WidenGEP represents a strided
/// access. If so, it creates recipes representing the base pointer and stride
-/// in element type, and returns a tuple of {base pointer, stride, element
+/// in element units, and returns a tuple of {base pointer, stride, element
/// type}. Otherwise, returns a tuple where all elements are nullptr.
static std::tuple<VPValue *, VPValue *, Type *>
determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
>From b9af2030fcb999289f53ea9e64ece7b194cc7111 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 7 Nov 2025 01:11:31 -0800
Subject: [PATCH 12/35] cast to VPWidenLoadRecipe directly for now. nfc
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4b25eaf396203..464a32a153359 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6608,31 +6608,31 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *MemR = dyn_cast<VPWidenMemoryRecipe>(&R);
+ auto *LoadR = dyn_cast<VPWidenLoadRecipe>(&R);
// TODO: Support strided store.
// TODO: Transform reverse access into strided access with -1 stride.
// TODO: Transform gather/scatter with uniform address into strided access
// with 0 stride.
// TODO: Transform interleave access into multiple strided accesses.
- if (!MemR || !isa<VPWidenLoadRecipe>(MemR) || MemR->isConsecutive())
+ if (!LoadR || LoadR->isConsecutive())
continue;
- auto *Ptr = dyn_cast<VPWidenGEPRecipe>(MemR->getAddr());
+ auto *Ptr = dyn_cast<VPWidenGEPRecipe>(LoadR->getAddr());
if (!Ptr)
continue;
- Instruction &Ingredient = MemR->getIngredient();
+ Instruction &Ingredient = LoadR->getIngredient();
auto IsProfitable = [&](ElementCount VF) -> bool {
Type *DataTy = toVectorTy(getLoadStoreType(&Ingredient), VF);
const Align Alignment = getLoadStoreAlignment(&Ingredient);
if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment))
return false;
- const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
+ const InstructionCost CurrentCost = LoadR->computeCost(VF, Ctx);
const InstructionCost StridedLoadStoreCost =
Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::experimental_vp_strided_load, DataTy,
- Ptr->getUnderlyingValue(), MemR->isMasked(), Alignment,
+ Ptr->getUnderlyingValue(), LoadR->isMasked(), Alignment,
&Ingredient),
Ctx.CostKind);
return StridedLoadStoreCost < CurrentCost;
@@ -6663,7 +6663,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
auto *NewPtr = new VPVectorPointerRecipe(
BasePtr, ElementTy, StrideInElement, Ptr->getGEPNoWrapFlags(),
Ptr->getDebugLoc());
- NewPtr->insertBefore(MemR);
+ NewPtr->insertBefore(LoadR);
const DataLayout &DL = Ingredient.getDataLayout();
TypeSize TS = DL.getTypeAllocSize(ElementTy);
@@ -6676,11 +6676,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
auto *ScaledStride =
new VPInstruction(Instruction::Mul, {StrideInElement, ScaleVPV},
VPRecipeWithIRFlags::WrapFlagsTy{false, false});
- ScaledStride->insertBefore(MemR);
+ ScaledStride->insertBefore(LoadR);
StrideInBytes = ScaledStride;
}
- auto *LoadR = cast<VPWidenLoadRecipe>(MemR);
auto *StridedLoad = new VPWidenStridedLoadRecipe(
*cast<LoadInst>(&Ingredient), NewPtr, StrideInBytes, &Plan.getVF(),
LoadR->getMask(), *LoadR, LoadR->getDebugLoc());
>From 8937ca538feaa6beb77516465fb7f22dc685607e Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 7 Nov 2025 01:14:10 -0800
Subject: [PATCH 13/35] replace getOrAddLiveIn(ConstantInt::get()) with
getConstantInt. nfc
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0a112f691f28c..06c28a17f6442 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6431,7 +6431,7 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
} else {
const DataLayout &DL = I->getDataLayout();
auto *StrideTy = DL.getIndexType(Ptr->getUnderlyingValue()->getType());
- VPValue *StrideOne = Plan.getOrAddLiveIn(ConstantInt::get(StrideTy, 1));
+ VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I), StrideOne,
Flags, VPI->getDebugLoc());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 464a32a153359..2ba8a76bc705a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6671,8 +6671,8 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
VPValue *StrideInBytes = StrideInElement;
// Scale the stride by the size of the indexed type.
if (TypeScale != 1) {
- VPValue *ScaleVPV = Plan.getOrAddLiveIn(ConstantInt::get(
- TypeInfo.inferScalarType(StrideInElement), TypeScale));
+ VPValue *ScaleVPV = Plan.getConstantInt(
+ TypeInfo.inferScalarType(StrideInElement), TypeScale);
auto *ScaledStride =
new VPInstruction(Instruction::Mul, {StrideInElement, ScaleVPV},
VPRecipeWithIRFlags::WrapFlagsTy{false, false});
>From 2dde97fc2d35b9b4b5dab6c60f31c1ba8bff9e76 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 7 Nov 2025 01:28:12 -0800
Subject: [PATCH 14/35] updated assertion message. nfc
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2ba8a76bc705a..022a4377ef54c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6653,11 +6653,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
determineBaseAndStride(Ptr);
// Skip if the memory access is not a strided access.
- if (!BasePtr) {
- assert(!StrideInElement && !ElementTy);
+ if (!BasePtr)
continue;
- }
- assert(StrideInElement && ElementTy);
+ assert(StrideInElement && ElementTy &&
+ "Can not get stride information for a strided access");
// Create a new vector pointer for strided access.
auto *NewPtr = new VPVectorPointerRecipe(
>From 83ac88b70c99979f968707e5a35cde78a0cdaf3e Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 10 Nov 2025 23:23:08 -0800
Subject: [PATCH 15/35] rebase and remove getLoadStoreAlignment call
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d8d1d53fe40e3..458f428d1f1a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3856,7 +3856,6 @@ void VPWidenLoadEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPWidenStridedLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getLoadStoreType(&Ingredient);
auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
- const Align Alignment = getLoadStoreAlignment(&Ingredient);
auto &Builder = State.Builder;
Value *Addr = State.get(getAddr(), /*IsScalar*/ true);
@@ -3885,7 +3884,6 @@ VPWidenStridedLoadRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
- const Align Alignment = getLoadStoreAlignment(&Ingredient);
return Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load, Ty,
Ptr, IsMasked, Alignment, &Ingredient),
>From 3e03edcb5a547023dd2ba8f6f9014a22091268b6 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 2 Dec 2025 07:38:14 -0800
Subject: [PATCH 16/35] Intro VPWidenMemIntrinsicRecipe
---
llvm/lib/Analysis/VectorUtils.cpp | 4 +
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 96 ++++++++++++++
.../Transforms/Vectorize/VPlanAnalysis.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 125 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 43 +++++-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 +
.../LoopVectorize/RISCV/strided-accesses.ll | 29 ++--
.../RISCV/tail-folding-gather-scatter.ll | 2 +-
.../RISCV/tail-folding-interleave.ll | 11 +-
10 files changed, 282 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 8b4d6d9652414..39a72d2bffa43 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -167,6 +167,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
return (ScalarOpdIdx == 2);
case Intrinsic::experimental_vp_splice:
return ScalarOpdIdx == 2 || ScalarOpdIdx == 4;
+ case Intrinsic::experimental_vp_strided_load:
+ return ScalarOpdIdx == 0 || ScalarOpdIdx == 1;
default:
return false;
}
@@ -204,6 +206,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::powi:
case Intrinsic::ldexp:
return OpdIdx == -1 || OpdIdx == 1;
+ case Intrinsic::experimental_vp_strided_load:
+ return OpdIdx == -1 || OpdIdx == 0 || OpdIdx == 1;
default:
return OpdIdx == -1;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 06c28a17f6442..49273bf5eba44 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3323,6 +3323,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPWidenCastSC:
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPWidenIntrinsicSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
case VPRecipeBase::VPWidenSC:
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 72bfba7935a69..cca208d5b4e2a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -440,6 +440,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPWidenCastSC,
VPWidenGEPSC,
VPWidenIntrinsicSC,
+ VPWidenMemIntrinsicSC,
VPWidenStridedLoadSC,
VPWidenLoadEVLSC,
VPWidenLoadSC,
@@ -645,6 +646,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
@@ -1981,6 +1983,98 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
#endif
};
+/// A recipe for widening vector memory intrinsics.
+class VPWidenMemIntrinsicRecipe : public VPRecipeBase,
+ public VPIRMetadata{
+ Instruction &Ingredient;
+
+ /// Alignment information for this memory access.
+ Align Alignment;
+
+ /// ID of the vector intrinsic to widen.
+ Intrinsic::ID VectorIntrinsicID;
+
+ /// Scalar return type of the intrinsic.
+ Type *ResultTy;
+
+ /// True if the intrinsic may read from memory.
+ bool MayReadFromMemory;
+
+ /// True if the intrinsic may read write to memory.
+ bool MayWriteToMemory;
+
+ /// True if the intrinsic may have side-effects.
+ bool MayHaveSideEffects;
+
+public:
+ // TODO: support StoreInst for strided store
+ VPWidenMemIntrinsicRecipe(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments,
+ const VPIRMetadata &MD = {},
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(VPRecipeBase::VPWidenMemIntrinsicSC, CallArguments, DL),
+ VPIRMetadata(MD), Ingredient(LI), Alignment(LI.getAlign()),
+ VectorIntrinsicID(VectorIntrinsicID), ResultTy(LI.getType()),
+ MayReadFromMemory(LI.mayReadFromMemory()),
+ MayHaveSideEffects(LI.mayHaveSideEffects()) {
+ new VPRecipeValue(this, &LI);
+ }
+
+ ~VPWidenMemIntrinsicRecipe() override = default;
+
+ VPWidenMemIntrinsicRecipe *clone() override {
+ return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(&Ingredient),
+ VectorIntrinsicID, operands(), *this,
+ getDebugLoc());
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemIntrinsicSC)
+
+ /// Produce a widened version of the vector memory intrinsic.
+ void execute(VPTransformState &State) override;
+
+ /// Return the cost of this vector memory intrinsic.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+ /// Return the ID of the intrinsic.
+ Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
+
+ /// Return the scalar return type of the intrinsic.
+ Type *getResultType() const { return ResultTy; }
+
+ /// Return to name of the intrinsic as string.
+ StringRef getIntrinsicName() const {
+ return Intrinsic::getBaseName(VectorIntrinsicID);
+ }
+
+ /// Returns true if the intrinsic may read from memory.
+ bool mayReadFromMemory() const { return MayReadFromMemory; }
+
+ /// Returns true if the intrinsic may write to memory.
+ bool mayWriteToMemory() const { return MayWriteToMemory; }
+
+ /// Returns true if the intrinsic may have side-effects.
+ bool mayHaveSideEffects() const { return MayHaveSideEffects; }
+
+ unsigned getMemoryPointerParamPos() const;
+
+ unsigned getMaskParamPos() const;
+
+ void setMask(VPValue *Mask) { setOperand(getMaskParamPos(), Mask); }
+
+ VPValue *getMask() const { return getOperand(getMaskParamPos()); }
+
+ bool usesFirstLaneOnly(const VPValue *Op) const override;
+
+protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for widening Call instructions using library calls.
class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
@@ -4191,6 +4285,8 @@ struct CastInfo<VPIRMetadata, VPRecipeBase *>
return cast<VPWidenCastRecipe>(R);
case VPRecipeBase::VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(R);
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
+ return cast<VPWidenMemIntrinsicRecipe>(R);
case VPRecipeBase::VPWidenCallSC:
return cast<VPWidenCallRecipe>(R);
case VPRecipeBase::VPReplicateSC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 7882187fa5314..f5b56d6b4b98c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -311,7 +311,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
})
// VPInstructionWithType must be handled before VPInstruction.
.Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
- VPWidenCastRecipe>(
+ VPWidenMemIntrinsicRecipe, VPWidenCastRecipe>(
[](const auto *R) { return R->getResultType(); })
.Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
VPWidenCallRecipe, VPWidenMemoryRecipe>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 458f428d1f1a0..a384ba5db1955 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -73,6 +73,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
->onlyReadsMemory();
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory();
+ case VPWidenMemIntrinsicSC:
+ return cast<VPWidenMemIntrinsicRecipe>(this)->mayWriteToMemory();
case VPActiveLaneMaskPHISC:
case VPCurrentIterationPHISC:
case VPBranchOnMaskSC:
@@ -127,6 +129,8 @@ bool VPRecipeBase::mayReadFromMemory() const {
->onlyWritesMemory();
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayReadFromMemory();
+ case VPWidenMemIntrinsicSC:
+ return cast<VPWidenMemIntrinsicRecipe>(this)->mayReadFromMemory();
case VPBranchOnMaskSC:
case VPDerivedIVSC:
case VPCurrentIterationPHISC:
@@ -186,6 +190,8 @@ bool VPRecipeBase::mayHaveSideEffects() const {
}
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayHaveSideEffects();
+ case VPWidenMemIntrinsicSC:
+ return cast<VPWidenMemIntrinsicRecipe>(this)->mayHaveSideEffects();
case VPBlendSC:
case VPReductionEVLSC:
case VPReductionSC:
@@ -2032,6 +2038,125 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
+unsigned VPWidenMemIntrinsicRecipe::getMemoryPointerParamPos() const {
+ if (auto Pos = VPIntrinsic::getMemoryPointerParamPos(VectorIntrinsicID))
+ return *Pos;
+
+ switch (VectorIntrinsicID) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_gather:
+ case Intrinsic::masked_expandload:
+ return 0;
+ case Intrinsic::masked_store:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_compressstore:
+ return 1;
+ default:
+ llvm_unreachable("unknown vector memory intrinsic");
+ }
+}
+
+unsigned VPWidenMemIntrinsicRecipe::getMaskParamPos() const {
+ if (auto Pos = VPIntrinsic::getMaskParamPos(VectorIntrinsicID))
+ return *Pos;
+
+ switch (VectorIntrinsicID) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_gather:
+ case Intrinsic::masked_expandload:
+ return 1;
+ case Intrinsic::masked_store:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_compressstore:
+ return 2;
+ default:
+ llvm_unreachable("unknown vector memory intrinsic");
+ }
+}
+
+void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
+ assert(State.VF.isVector() && "not widening");
+
+ SmallVector<Type *, 2> TysForDecl;
+ // Add return type if intrinsic is overloaded on it.
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1, State.TTI))
+ TysForDecl.push_back(VectorType::get(getResultType(), State.VF));
+ SmallVector<Value *, 4> Args;
+ for (const auto &I : enumerate(operands())) {
+ // Some intrinsics have a scalar argument - don't replace it with a
+ // vector.
+ Value *Arg;
+ if (isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index(),
+ State.TTI))
+ Arg = State.get(I.value(), VPLane(0));
+ else
+ Arg = State.get(I.value(), usesFirstLaneOnly(I.value()));
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index(),
+ State.TTI))
+ TysForDecl.push_back(Arg->getType());
+ Args.push_back(Arg);
+ }
+
+ // Use vector version of the intrinsic.
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ Function *VectorF =
+ Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF &&
+ "Can't retrieve vector intrinsic or vector-predication intrinsics.");
+
+ CallInst *MemI = State.Builder.CreateCall(VectorF, Args);
+ MemI->addParamAttr(
+ getMemoryPointerParamPos(),
+ Attribute::getWithAlignment(MemI->getContext(), Alignment));
+ applyMetadata(*MemI);
+
+ if (!MemI->getType()->isVoidTy())
+ State.set(getVPSingleValue(), MemI);
+}
+
+InstructionCost
+VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
+ const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ return Ctx.TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(VectorIntrinsicID, Ty, Ptr,
+ match(getMask(), m_True()), Alignment,
+ &Ingredient),
+ Ctx.CostKind);
+}
+
+// Copy from VPWidenIntrinsicRecipe::usesFirstLaneOnly.
+bool VPWidenMemIntrinsicRecipe::usesFirstLaneOnly(const VPValue *Op) const {
+ assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
+ return all_of(enumerate(operands()), [this, &Op](const auto &X) {
+ auto [Idx, V] = X;
+ return V != Op || isVectorIntrinsicWithScalarOpAtArg(getVectorIntrinsicID(),
+ Idx, nullptr);
+ });
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPWidenMemIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-MEM-INTRINSIC ";
+ if (ResultTy->isVoidTy()) {
+ O << "void ";
+ } else {
+ getVPSingleValue()->printAsOperand(O, SlotTracker);
+ O << " = ";
+ }
+
+ O << "call ";
+ O << getIntrinsicName() << "(";
+
+ interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
+ O << ")";
+}
+#endif
+
void VPHistogramRecipe::execute(VPTransformState &State) {
IRBuilderBase &Builder = State.Builder;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 022a4377ef54c..08031155ea900 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2935,6 +2935,18 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
TypeInfo.inferScalarType(LoadR), {}, {}, DL);
}
+ if (auto *MI = dyn_cast<VPWidenMemIntrinsicRecipe>(&CurRecipe))
+ if (MI->getVectorIntrinsicID() == Intrinsic::experimental_vp_strided_load &&
+ match(MI->getMask(), m_RemoveMask(HeaderMask, Mask))) {
+ VPWidenMemIntrinsicRecipe *NewMI = MI->clone();
+ if (Mask)
+ NewMI->setMask(Mask);
+ else
+ NewMI->setMask(Plan->getTrue());
+ NewMI->setOperand(3, &EVL);
+ return NewMI;
+ }
+
if (auto *StridedL = dyn_cast<VPWidenStridedLoadRecipe>(&CurRecipe))
if (StridedL->isMasked() &&
match(StridedL->getMask(), m_RemoveMask(HeaderMask, Mask)))
@@ -3068,8 +3080,13 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
assert(
all_of(
Plan.getVF().users(),
- IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe, VPWidenStridedLoadRecipe>) &&
+ [&LoopRegion](VPUser *U) {
+ auto *R = cast<VPRecipeBase>(U);
+ return (R->getParent()->getParent() != LoopRegion) ||
+ isa<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+ VPWidenIntOrFpInductionRecipe, VPWidenStridedLoadRecipe>(
+ R);
+ }) &&
"User of VF that we can't transform to EVL.");
Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
@@ -6605,6 +6622,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
DenseMap<VPWidenGEPRecipe *, std::tuple<VPValue *, VPValue *, Type *>>
StrideCache;
SmallVector<VPWidenMemoryRecipe *> ToErase;
+ VPValue *I32VF = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -6658,6 +6676,14 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
assert(StrideInElement && ElementTy &&
"Can not get stride information for a strided access");
+ // Add VF of i32 version for EVL.
+ if (!I32VF) {
+ VPBuilder Builder(Plan.getVectorPreheader());
+ I32VF = Builder.createScalarZExtOrTrunc(
+ &Plan.getVF(), Type::getInt32Ty(Plan.getContext()),
+ TypeInfo.inferScalarType(&Plan.getVF()), DebugLoc::getUnknown());
+ }
+
// Create a new vector pointer for strided access.
auto *NewPtr = new VPVectorPointerRecipe(
BasePtr, ElementTy, StrideInElement, Ptr->getGEPNoWrapFlags(),
@@ -6679,11 +6705,16 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
StrideInBytes = ScaledStride;
}
- auto *StridedLoad = new VPWidenStridedLoadRecipe(
- *cast<LoadInst>(&Ingredient), NewPtr, StrideInBytes, &Plan.getVF(),
- LoadR->getMask(), *LoadR, LoadR->getDebugLoc());
+ VPValue *Mask;
+ if (VPValue *LoadMask = LoadR->getMask())
+ Mask = LoadMask;
+ else
+ Mask = Plan.getTrue();
+ auto *StridedLoad = new VPWidenMemIntrinsicRecipe(
+ *cast<LoadInst>(&Ingredient), Intrinsic::experimental_vp_strided_load,
+ {NewPtr, StrideInBytes, Mask, I32VF}, *LoadR, LoadR->getDebugLoc());
StridedLoad->insertBefore(LoadR);
- LoadR->replaceAllUsesWith(StridedLoad);
+ LoadR->replaceAllUsesWith(StridedLoad->getVPSingleValue());
ToErase.push_back(LoadR);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8f9806adf774e..dd2cb1cf094d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -51,6 +51,7 @@ class LLVM_ABI_FOR_TEST VPValue {
friend struct VPSymbolicValue;
friend class VPRecipeValue;
friend class VPRegionValue;
+ friend class VPWidenMemIntrinsicRecipe;
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index a6bada6bc6332..3925aa548831f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -55,6 +55,7 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-UF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-UF2-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 [[N_MOD_VF]]
; CHECK-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[TMP6]]
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-UF2-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2: [[VECTOR_BODY]]:
@@ -69,10 +70,8 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 3
; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
-; CHECK-UF2-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
-; CHECK-UF2-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP20]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP17]])
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP20]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; CHECK-UF2-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-UF2-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], splat (i32 1)
; CHECK-UF2-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true))
@@ -167,6 +166,7 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; CHECK-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; CHECK-UF2-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 6
+; CHECK-UF2-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-UF2-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i64> [[BROADCAST_SPLAT]], splat (i64 6)
; CHECK-UF2-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-UF2-NEXT: [[TMP8:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP7]], splat (i64 64)
@@ -181,10 +181,8 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[STEP_ADD]]
; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 6
; CHECK-UF2-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
-; CHECK-UF2-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
-; CHECK-UF2-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
; CHECK-UF2-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-UF2-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], splat (i32 1)
; CHECK-UF2-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true))
@@ -198,11 +196,11 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-UF2: [[SCALAR_PH]]:
; CHECK-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-UF2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-UF2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-UF2-NEXT: br label %[[LOOP:.*]]
; CHECK-UF2: [[LOOP]]:
; CHECK-UF2-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXTI:%.*]], %[[LOOP]] ]
-; CHECK-UF2-NEXT: [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-UF2-NEXT: [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], %[[LOOP]] ]
; CHECK-UF2-NEXT: [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
; CHECK-UF2-NEXT: [[X0:%.*]] = load i32, ptr [[Q0]], align 4
; CHECK-UF2-NEXT: [[Y0:%.*]] = add i32 [[X0]], 1
@@ -905,6 +903,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 1
; STRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP30]]
; STRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; STRIDED-UF2-NEXT: [[TMP44:%.*]] = trunc i64 [[TMP29]] to i32
; STRIDED-UF2-NEXT: [[TMP32:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT10]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -920,10 +919,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP35]]
; STRIDED-UF2-NEXT: [[TMP42:%.*]] = mul i64 [[TMP29]], [[STRIDE]]
; STRIDED-UF2-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP36]], i64 [[TMP42]]
-; STRIDED-UF2-NEXT: [[TMP44:%.*]] = trunc i64 [[TMP29]] to i32
; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP36]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP44]]), !alias.scope [[META8:![0-9]+]]
-; STRIDED-UF2-NEXT: [[TMP45:%.*]] = trunc i64 [[TMP29]] to i32
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP43]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP45]]), !alias.scope [[META8]]
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP43]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP44]]), !alias.scope [[META8]]
; STRIDED-UF2-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER12]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP33]]
@@ -1365,15 +1362,14 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; NOSTRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; NOSTRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOSTRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
; NOSTRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED-UF2: [[VECTOR_BODY]]:
; NOSTRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
; NOSTRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
-; NOSTRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
-; NOSTRIDED-UF2-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
-; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
; NOSTRIDED-UF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i64 [[TMP3]]
; NOSTRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP9]], align 8
@@ -1436,15 +1432,14 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; STRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; STRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; STRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
; STRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED-UF2: [[VECTOR_BODY]]:
; STRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
; STRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
-; STRIDED-UF2-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
-; STRIDED-UF2-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; STRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
; STRIDED-UF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i64 [[TMP3]]
; STRIDED-UF2-NEXT: store <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP9]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
index be4300f2c3619..15b2cb0d0681c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
@@ -44,11 +44,11 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
; NO-VP-NEXT: br label [[FOR_BODY1:%.*]]
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV1]]
-; NO-VP-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
; NO-VP-NEXT: [[WIDE_STRIDED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[ARRAYIDX3]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP6]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x float> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 7236d0265216c..e625d0687e4bc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -602,26 +602,23 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[INDEX]]
; NO-VP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP7]], i32 0
-; NO-VP-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 1
-; NO-VP-NEXT: [[TMP23:%.*]] = trunc i64 [[TMP5]] to i32
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP23]])
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]
; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 2
-; NO-VP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP5]] to i32
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]
; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 3
-; NO-VP-NEXT: [[TMP26:%.*]] = trunc i64 [[TMP5]] to i32
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP26]])
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; NO-VP-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
>From b20e216fded2f89e90869c92609f0755f2e9eef9 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 17:52:55 -0800
Subject: [PATCH 17/35] Inherit from VPWidenIntrinsicRecipe
---
llvm/lib/Transforms/Vectorize/VPlan.h | 129 ++++++++----------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 104 ++++----------
llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 -
4 files changed, 82 insertions(+), 154 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index cca208d5b4e2a..edb5412c7820c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -631,6 +631,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPWidenCastSC:
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPWidenIntrinsicSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
case VPRecipeBase::VPWidenSC:
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPPredInstPHISC:
@@ -646,7 +647,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
case VPRecipeBase::VPIRInstructionSC:
- case VPRecipeBase::VPWidenMemIntrinsicSC:
case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
@@ -1124,6 +1124,7 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
R->getVPRecipeID() == VPRecipeBase::VPWidenCallSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenCastSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenIntrinsicSC ||
+ R->getVPRecipeID() == VPRecipeBase::VPWidenMemIntrinsicSC ||
R->getVPRecipeID() == VPRecipeBase::VPReductionSC ||
R->getVPRecipeID() == VPRecipeBase::VPReductionEVLSC ||
R->getVPRecipeID() == VPRecipeBase::VPReplicateSC ||
@@ -1901,6 +1902,29 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// True if the intrinsic may have side-effects.
bool MayHaveSideEffects;
+protected:
+ VPWidenIntrinsicRecipe(const unsigned char SC,
+ Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments, Type *Ty,
+ const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeWithIRFlags(SC, CallArguments, Flags, DL),
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ ResultTy(Ty) {
+ LLVMContext &Ctx = Ty->getContext();
+ AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
+ MemoryEffects ME = Attrs.getMemoryEffects();
+ MayReadFromMemory = !ME.onlyWritesMemory();
+ MayWriteToMemory = !ME.onlyReadsMemory();
+ MayHaveSideEffects = MayWriteToMemory ||
+ !Attrs.hasAttribute(Attribute::NoUnwind) ||
+ !Attrs.hasAttribute(Attribute::WillReturn);
+ }
+
+ /// Helper function to produce the widened intrinsic call.
+ CallInst *createVectorCall(VPTransformState &State);
+
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
@@ -1921,19 +1945,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
- Flags, DL),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty) {
- LLVMContext &Ctx = Ty->getContext();
- AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
- MemoryEffects ME = Attrs.getMemoryEffects();
- MayReadFromMemory = !ME.onlyWritesMemory();
- MayWriteToMemory = !ME.onlyReadsMemory();
- MayHaveSideEffects = MayWriteToMemory ||
- !Attrs.hasAttribute(Attribute::NoUnwind) ||
- !Attrs.hasAttribute(Attribute::WillReturn);
- }
+ : VPWidenIntrinsicRecipe(VPRecipeBase::VPWidenIntrinsicSC,
+ VectorIntrinsicID, CallArguments, Ty, Flags,
+ Metadata, DL) {}
~VPWidenIntrinsicRecipe() override = default;
@@ -1946,7 +1960,24 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
*this, *this, getDebugLoc());
}
- VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntrinsicSC)
+ static inline bool classof(const VPRecipeBase *R) {
+ return R->getVPRecipeID() == VPRecipeBase::VPWidenIntrinsicSC ||
+ R->getVPRecipeID() == VPRecipeBase::VPWidenMemIntrinsicSC;
+ }
+
+ static inline bool classof(const VPUser *U) {
+ auto *R = dyn_cast<VPRecipeBase>(U);
+ return R && classof(R);
+ }
+
+ static inline bool classof(const VPValue *V) {
+ auto *R = V->getDefiningRecipe();
+ return R && classof(R);
+ }
+
+ static inline bool classof(const VPSingleDefRecipe *R) {
+ return classof(static_cast<const VPRecipeBase *>(R));
+ }
/// Produce a widened version of the vector intrinsic.
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
@@ -1984,48 +2015,29 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
};
/// A recipe for widening vector memory intrinsics.
-class VPWidenMemIntrinsicRecipe : public VPRecipeBase,
- public VPIRMetadata{
- Instruction &Ingredient;
-
+class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
/// Alignment information for this memory access.
Align Alignment;
- /// ID of the vector intrinsic to widen.
- Intrinsic::ID VectorIntrinsicID;
-
- /// Scalar return type of the intrinsic.
- Type *ResultTy;
-
- /// True if the intrinsic may read from memory.
- bool MayReadFromMemory;
-
- /// True if the intrinsic may read write to memory.
- bool MayWriteToMemory;
-
- /// True if the intrinsic may have side-effects.
- bool MayHaveSideEffects;
-
public:
// TODO: support StoreInst for strided store
VPWidenMemIntrinsicRecipe(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments,
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(VPRecipeBase::VPWidenMemIntrinsicSC, CallArguments, DL),
- VPIRMetadata(MD), Ingredient(LI), Alignment(LI.getAlign()),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(LI.getType()),
- MayReadFromMemory(LI.mayReadFromMemory()),
- MayHaveSideEffects(LI.mayHaveSideEffects()) {
- new VPRecipeValue(this, &LI);
+ : VPWidenIntrinsicRecipe(VPRecipeBase::VPWidenMemIntrinsicSC,
+ VectorIntrinsicID, CallArguments, LI.getType(),
+ {}, MD, DL),
+ Alignment(LI.getAlign()) {
+ setUnderlyingValue(&LI);
}
~VPWidenMemIntrinsicRecipe() override = default;
VPWidenMemIntrinsicRecipe *clone() override {
- return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(&Ingredient),
- VectorIntrinsicID, operands(), *this,
- getDebugLoc());
+ return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(getUnderlyingInstr()),
+ getVectorIntrinsicID(), operands(),
+ *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemIntrinsicSC)
@@ -2037,42 +2049,15 @@ class VPWidenMemIntrinsicRecipe : public VPRecipeBase,
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- /// Return the ID of the intrinsic.
- Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
-
- /// Return the scalar return type of the intrinsic.
- Type *getResultType() const { return ResultTy; }
-
- /// Return to name of the intrinsic as string.
- StringRef getIntrinsicName() const {
- return Intrinsic::getBaseName(VectorIntrinsicID);
- }
-
- /// Returns true if the intrinsic may read from memory.
- bool mayReadFromMemory() const { return MayReadFromMemory; }
-
- /// Returns true if the intrinsic may write to memory.
- bool mayWriteToMemory() const { return MayWriteToMemory; }
-
- /// Returns true if the intrinsic may have side-effects.
- bool mayHaveSideEffects() const { return MayHaveSideEffects; }
-
+ /// Return the index of pointer parameter.
unsigned getMemoryPointerParamPos() const;
+ /// Return the index of mask parameter.
unsigned getMaskParamPos() const;
void setMask(VPValue *Mask) { setOperand(getMaskParamPos(), Mask); }
VPValue *getMask() const { return getOperand(getMaskParamPos()); }
-
- bool usesFirstLaneOnly(const VPValue *Op) const override;
-
-protected:
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
};
/// A recipe for widening Call instructions using library calls.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index f5b56d6b4b98c..7882187fa5314 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -311,7 +311,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
})
// VPInstructionWithType must be handled before VPInstruction.
.Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
- VPWidenMemIntrinsicRecipe, VPWidenCastRecipe>(
+ VPWidenCastRecipe>(
[](const auto *R) { return R->getResultType(); })
.Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
VPWidenCallRecipe, VPWidenMemoryRecipe>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a384ba5db1955..dd374c5c42b83 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -71,10 +71,9 @@ bool VPRecipeBase::mayWriteToMemory() const {
return !cast<VPWidenCallRecipe>(this)
->getCalledScalarFunction()
->onlyReadsMemory();
+ case VPWidenMemIntrinsicSC:
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory();
- case VPWidenMemIntrinsicSC:
- return cast<VPWidenMemIntrinsicRecipe>(this)->mayWriteToMemory();
case VPActiveLaneMaskPHISC:
case VPCurrentIterationPHISC:
case VPBranchOnMaskSC:
@@ -127,10 +126,9 @@ bool VPRecipeBase::mayReadFromMemory() const {
return !cast<VPWidenCallRecipe>(this)
->getCalledScalarFunction()
->onlyWritesMemory();
+ case VPWidenMemIntrinsicSC:
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayReadFromMemory();
- case VPWidenMemIntrinsicSC:
- return cast<VPWidenMemIntrinsicRecipe>(this)->mayReadFromMemory();
case VPBranchOnMaskSC:
case VPDerivedIVSC:
case VPCurrentIterationPHISC:
@@ -188,10 +186,9 @@ bool VPRecipeBase::mayHaveSideEffects() const {
Function *Fn = cast<VPWidenCallRecipe>(this)->getCalledScalarFunction();
return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn();
}
+ case VPWidenMemIntrinsicSC:
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayHaveSideEffects();
- case VPWidenMemIntrinsicSC:
- return cast<VPWidenMemIntrinsicRecipe>(this)->mayHaveSideEffects();
case VPBlendSC:
case VPReductionEVLSC:
case VPReductionSC:
@@ -1897,9 +1894,7 @@ void VPWidenCallRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
- assert(State.VF.isVector() && "not widening");
-
+CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
SmallVector<Type *, 2> TysForDecl;
// Add return type if intrinsic is overloaded on it.
if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1,
@@ -1935,7 +1930,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
assert(VectorF &&
"Can't retrieve vector intrinsic or vector-predication intrinsics.");
- auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
+ auto *CI = dyn_cast_or_null<CallInst>(getUnderlyingValue());
SmallVector<OperandBundleDef, 1> OpBundles;
if (CI)
CI->getOperandBundlesAsDefs(OpBundles);
@@ -1945,6 +1940,12 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
applyFlags(*V);
applyMetadata(*V);
+ return V;
+}
+
+void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
+ assert(State.VF.isVector() && "not widening");
+ CallInst *V = createVectorCall(State);
if (!V->getType()->isVoidTy())
State.set(this, V);
}
@@ -2039,10 +2040,11 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
unsigned VPWidenMemIntrinsicRecipe::getMemoryPointerParamPos() const {
- if (auto Pos = VPIntrinsic::getMemoryPointerParamPos(VectorIntrinsicID))
+ Intrinsic::ID IID = getVectorIntrinsicID();
+ if (auto Pos = VPIntrinsic::getMemoryPointerParamPos(IID))
return *Pos;
- switch (VectorIntrinsicID) {
+ switch (IID) {
case Intrinsic::masked_load:
case Intrinsic::masked_gather:
case Intrinsic::masked_expandload:
@@ -2057,10 +2059,11 @@ unsigned VPWidenMemIntrinsicRecipe::getMemoryPointerParamPos() const {
}
unsigned VPWidenMemIntrinsicRecipe::getMaskParamPos() const {
- if (auto Pos = VPIntrinsic::getMaskParamPos(VectorIntrinsicID))
+ Intrinsic::ID IID = getVectorIntrinsicID();
+ if (auto Pos = VPIntrinsic::getMaskParamPos(IID))
return *Pos;
- switch (VectorIntrinsicID) {
+ switch (IID) {
case Intrinsic::masked_load:
case Intrinsic::masked_gather:
case Intrinsic::masked_expandload:
@@ -2076,87 +2079,28 @@ unsigned VPWidenMemIntrinsicRecipe::getMaskParamPos() const {
void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
-
- SmallVector<Type *, 2> TysForDecl;
- // Add return type if intrinsic is overloaded on it.
- if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1, State.TTI))
- TysForDecl.push_back(VectorType::get(getResultType(), State.VF));
- SmallVector<Value *, 4> Args;
- for (const auto &I : enumerate(operands())) {
- // Some intrinsics have a scalar argument - don't replace it with a
- // vector.
- Value *Arg;
- if (isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index(),
- State.TTI))
- Arg = State.get(I.value(), VPLane(0));
- else
- Arg = State.get(I.value(), usesFirstLaneOnly(I.value()));
- if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index(),
- State.TTI))
- TysForDecl.push_back(Arg->getType());
- Args.push_back(Arg);
- }
-
- // Use vector version of the intrinsic.
- Module *M = State.Builder.GetInsertBlock()->getModule();
- Function *VectorF =
- Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF &&
- "Can't retrieve vector intrinsic or vector-predication intrinsics.");
-
- CallInst *MemI = State.Builder.CreateCall(VectorF, Args);
+ CallInst *MemI = createVectorCall(State);
MemI->addParamAttr(
getMemoryPointerParamPos(),
Attribute::getWithAlignment(MemI->getContext(), Alignment));
- applyMetadata(*MemI);
if (!MemI->getType()->isVoidTy())
- State.set(getVPSingleValue(), MemI);
+ State.set(this, MemI);
}
InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
- Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
- const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ const Instruction *Ingredient = getUnderlyingInstr();
+ Type *Ty = toVectorTy(getLoadStoreType(Ingredient), VF);
+ const Value *Ptr = getLoadStorePointerOperand(Ingredient);
return Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(VectorIntrinsicID, Ty, Ptr,
+ MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, Ptr,
match(getMask(), m_True()), Alignment,
- &Ingredient),
+ Ingredient),
Ctx.CostKind);
}
-// Copy from VPWidenIntrinsicRecipe::usesFirstLaneOnly.
-bool VPWidenMemIntrinsicRecipe::usesFirstLaneOnly(const VPValue *Op) const {
- assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
- return all_of(enumerate(operands()), [this, &Op](const auto &X) {
- auto [Idx, V] = X;
- return V != Op || isVectorIntrinsicWithScalarOpAtArg(getVectorIntrinsicID(),
- Idx, nullptr);
- });
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPWidenMemIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "WIDEN-MEM-INTRINSIC ";
- if (ResultTy->isVoidTy()) {
- O << "void ";
- } else {
- getVPSingleValue()->printAsOperand(O, SlotTracker);
- O << " = ";
- }
-
- O << "call ";
- O << getIntrinsicName() << "(";
-
- interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) {
- Op->printAsOperand(O, SlotTracker);
- });
- O << ")";
-}
-#endif
-
void VPHistogramRecipe::execute(VPTransformState &State) {
IRBuilderBase &Builder = State.Builder;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index dd2cb1cf094d5..8f9806adf774e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -51,7 +51,6 @@ class LLVM_ABI_FOR_TEST VPValue {
friend struct VPSymbolicValue;
friend class VPRecipeValue;
friend class VPRegionValue;
- friend class VPWidenMemIntrinsicRecipe;
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
>From 28fc91a4c9775955b994b17c234534a175f82921 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 18:20:05 -0800
Subject: [PATCH 18/35] narrow to Intrinsic::experimental_vp_strided_load
---
llvm/lib/Transforms/Vectorize/VPlan.h | 20 +++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 51 ++-----------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +--
3 files changed, 14 insertions(+), 63 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index edb5412c7820c..e6f06df917668 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2021,13 +2021,12 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
public:
// TODO: support StoreInst for strided store
- VPWidenMemIntrinsicRecipe(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
- ArrayRef<VPValue *> CallArguments,
+ VPWidenMemIntrinsicRecipe(LoadInst &LI, ArrayRef<VPValue *> CallArguments,
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPWidenIntrinsicRecipe(VPRecipeBase::VPWidenMemIntrinsicSC,
- VectorIntrinsicID, CallArguments, LI.getType(),
- {}, MD, DL),
+ Intrinsic::experimental_vp_strided_load,
+ CallArguments, LI.getType(), {}, MD, DL),
Alignment(LI.getAlign()) {
setUnderlyingValue(&LI);
}
@@ -2036,8 +2035,7 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
VPWidenMemIntrinsicRecipe *clone() override {
return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(getUnderlyingInstr()),
- getVectorIntrinsicID(), operands(),
- *this, getDebugLoc());
+ operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemIntrinsicSC)
@@ -2049,15 +2047,9 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- /// Return the index of pointer parameter.
- unsigned getMemoryPointerParamPos() const;
+ void setMask(VPValue *Mask) { setOperand(2, Mask); }
- /// Return the index of mask parameter.
- unsigned getMaskParamPos() const;
-
- void setMask(VPValue *Mask) { setOperand(getMaskParamPos(), Mask); }
-
- VPValue *getMask() const { return getOperand(getMaskParamPos()); }
+ VPValue *getMask() const { return getOperand(2); }
};
/// A recipe for widening Call instructions using library calls.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dd374c5c42b83..032ad4b3c5021 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1895,6 +1895,8 @@ void VPWidenCallRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
+ assert(State.VF.isVector() && "not widening");
+
SmallVector<Type *, 2> TysForDecl;
// Add return type if intrinsic is overloaded on it.
if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1,
@@ -1944,7 +1946,6 @@ CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
}
void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
- assert(State.VF.isVector() && "not widening");
CallInst *V = createVectorCall(State);
if (!V->getType()->isVoidTy())
State.set(this, V);
@@ -2039,60 +2040,18 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
-unsigned VPWidenMemIntrinsicRecipe::getMemoryPointerParamPos() const {
- Intrinsic::ID IID = getVectorIntrinsicID();
- if (auto Pos = VPIntrinsic::getMemoryPointerParamPos(IID))
- return *Pos;
-
- switch (IID) {
- case Intrinsic::masked_load:
- case Intrinsic::masked_gather:
- case Intrinsic::masked_expandload:
- return 0;
- case Intrinsic::masked_store:
- case Intrinsic::masked_scatter:
- case Intrinsic::masked_compressstore:
- return 1;
- default:
- llvm_unreachable("unknown vector memory intrinsic");
- }
-}
-
-unsigned VPWidenMemIntrinsicRecipe::getMaskParamPos() const {
- Intrinsic::ID IID = getVectorIntrinsicID();
- if (auto Pos = VPIntrinsic::getMaskParamPos(IID))
- return *Pos;
-
- switch (IID) {
- case Intrinsic::masked_load:
- case Intrinsic::masked_gather:
- case Intrinsic::masked_expandload:
- return 1;
- case Intrinsic::masked_store:
- case Intrinsic::masked_scatter:
- case Intrinsic::masked_compressstore:
- return 2;
- default:
- llvm_unreachable("unknown vector memory intrinsic");
- }
-}
-
void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
- assert(State.VF.isVector() && "not widening");
CallInst *MemI = createVectorCall(State);
MemI->addParamAttr(
- getMemoryPointerParamPos(),
- Attribute::getWithAlignment(MemI->getContext(), Alignment));
-
- if (!MemI->getType()->isVoidTy())
- State.set(this, MemI);
+ 0, Attribute::getWithAlignment(MemI->getContext(), Alignment));
+ State.set(this, MemI);
}
InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ Type *Ty = toVectorTy(getResultType(), VF);
const Instruction *Ingredient = getUnderlyingInstr();
- Type *Ty = toVectorTy(getLoadStoreType(Ingredient), VF);
const Value *Ptr = getLoadStorePointerOperand(Ingredient);
return Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, Ptr,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 08031155ea900..db849bb449caa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6711,10 +6711,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
else
Mask = Plan.getTrue();
auto *StridedLoad = new VPWidenMemIntrinsicRecipe(
- *cast<LoadInst>(&Ingredient), Intrinsic::experimental_vp_strided_load,
- {NewPtr, StrideInBytes, Mask, I32VF}, *LoadR, LoadR->getDebugLoc());
+ *cast<LoadInst>(&Ingredient), {NewPtr, StrideInBytes, Mask, I32VF},
+ *LoadR, LoadR->getDebugLoc());
StridedLoad->insertBefore(LoadR);
- LoadR->replaceAllUsesWith(StridedLoad->getVPSingleValue());
+ LoadR->replaceAllUsesWith(StridedLoad);
ToErase.push_back(LoadR);
}
>From 8ace2114bb29150ca50a34af39f8754b5adfce3a Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 19:35:12 -0800
Subject: [PATCH 19/35] pattern matching
Intrinsic::experimental_vp_strided_load
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 24 ++++++++++---------
3 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e6f06df917668..62813f3d31fa3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2046,10 +2046,6 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
/// Return the cost of this vector memory intrinsic.
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
-
- void setMask(VPValue *Mask) { setOperand(2, Mask); }
-
- VPValue *getMask() const { return getOperand(2); }
};
/// A recipe for widening Call instructions using library calls.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 032ad4b3c5021..9b91cb1235cda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2055,7 +2055,7 @@ VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
const Value *Ptr = getLoadStorePointerOperand(Ingredient);
return Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, Ptr,
- match(getMask(), m_True()), Alignment,
+ match(getOperand(2), m_True()), Alignment,
Ingredient),
Ctx.CostKind);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index db849bb449caa..a7176e9e83c17 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2935,17 +2935,19 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
TypeInfo.inferScalarType(LoadR), {}, {}, DL);
}
- if (auto *MI = dyn_cast<VPWidenMemIntrinsicRecipe>(&CurRecipe))
- if (MI->getVectorIntrinsicID() == Intrinsic::experimental_vp_strided_load &&
- match(MI->getMask(), m_RemoveMask(HeaderMask, Mask))) {
- VPWidenMemIntrinsicRecipe *NewMI = MI->clone();
- if (Mask)
- NewMI->setMask(Mask);
- else
- NewMI->setMask(Plan->getTrue());
- NewMI->setOperand(3, &EVL);
- return NewMI;
- }
+ VPValue *Stride;
+ if (match(&CurRecipe, m_Intrinsic<Intrinsic::experimental_vp_strided_load>(
+ m_VPValue(Addr), m_VPValue(Stride),
+ m_RemoveMask(HeaderMask, Mask),
+ m_VPInstruction<Instruction::Trunc>(
+ m_Specific(&Plan->getVF()))))) {
+ auto *I = cast<VPWidenIntrinsicRecipe>(&CurRecipe);
+ if (!Mask)
+ Mask = Plan->getTrue();
+ return new VPWidenMemIntrinsicRecipe(
+ *cast<LoadInst>(I->getUnderlyingInstr()), {Addr, Stride, Mask, &EVL},
+ *I, I->getDebugLoc());
+ }
if (auto *StridedL = dyn_cast<VPWidenStridedLoadRecipe>(&CurRecipe))
if (StridedL->isMasked() &&
>From 83ef7b0ea2328c3ab3ab3d292cba506785256135 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 22:57:25 -0800
Subject: [PATCH 20/35] assert scalable and vector indexed type
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a7176e9e83c17..b02c58d152c4a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6594,8 +6594,9 @@ determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
return {nullptr, nullptr, nullptr};
Type *ElementTy = WidenGEP->getIndexedType(*VarIndex);
- if (ElementTy->isScalableTy() || ElementTy->isStructTy() ||
- ElementTy->isVectorTy())
+ assert(!ElementTy->isScalableTy() && !ElementTy->isVectorTy() &&
+ "Unexpected indexed type");
+ if (ElementTy->isStructTy())
return {nullptr, nullptr, nullptr};
unsigned VarOp = *VarIndex + 1;
>From 898930da096f274027c68ab686287997cd1f6317 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 23:09:23 -0800
Subject: [PATCH 21/35] remove unused VPWidenStridedLoadRecipe
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 56 +------------------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 6 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 54 ------------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 27 +++------
5 files changed, 12 insertions(+), 135 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49273bf5eba44..c27b441fae398 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3232,8 +3232,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
.Case([](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
.Case(
[](const VPWidenStoreRecipe *R) { return Instruction::Store; })
- .Case<VPWidenLoadRecipe, VPWidenStridedLoadRecipe>(
- [](const auto *R) { return Instruction::Load; })
+ .Case([](const VPWidenLoadRecipe *R) { return Instruction::Load; })
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
[](const auto *R) { return Instruction::Call; })
.Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
@@ -3334,7 +3333,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPReductionPHISC:
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
- case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 62813f3d31fa3..12ffeff044ae9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -441,7 +441,6 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPWidenGEPSC,
VPWidenIntrinsicSC,
VPWidenMemIntrinsicSC,
- VPWidenStridedLoadSC,
VPWidenLoadEVLSC,
VPWidenLoadSC,
VPWidenStoreEVLSC,
@@ -647,7 +646,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
case VPRecipeBase::VPIRInstructionSC:
- case VPRecipeBase::VPWidenStridedLoadSC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
@@ -3624,8 +3622,7 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
return R->getVPRecipeID() == VPRecipeBase::VPWidenLoadSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenStoreSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenLoadEVLSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenStridedLoadSC;
+ R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC;
}
static inline bool classof(const VPUser *U) {
@@ -3746,57 +3743,6 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
#endif
};
-/// A recipe for strided load operations, using the base address, stride, VF,
-/// and an optional mask. This recipe will generate a vp.strided.load intrinsic
-/// call to represent memory accesses with a fixed stride.
-struct VPWidenStridedLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
- VPWidenStridedLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Stride,
- VPValue *VF, VPValue *Mask,
- const VPIRMetadata &Metadata, DebugLoc DL)
- : VPWidenMemoryRecipe(VPRecipeBase::VPWidenStridedLoadSC, Load,
- {Addr, Stride, VF},
- /*Consecutive=*/false, Metadata, DL),
- VPRecipeValue(this, &Load) {
- setMask(Mask);
- }
-
- VPWidenStridedLoadRecipe *clone() override {
- return new VPWidenStridedLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getStride(), getVF(), getMask(), *this,
- getDebugLoc());
- }
-
- VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStridedLoadSC);
-
- /// Return the stride operand.
- VPValue *getStride() const { return getOperand(1); }
-
- /// Return the VF operand.
- VPValue *getVF() const { return getOperand(2); }
-
- /// Generate a strided load.
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPWidenStridedLoadRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- /// Returns true if the recipe only uses the first lane of operand \p Op.
- bool usesFirstLaneOnly(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- // All operands except the mask are only used for the first lane.
- return Op == getAddr() || Op == getStride() || Op == getVF();
- }
-};
-
/// A recipe for widening store operations, using the stored value, the address
/// to store to and an optional mask.
struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 7882187fa5314..103dff1889a6a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -211,10 +211,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) {
}
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) {
- assert(
- (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe, VPWidenStridedLoadRecipe>(
- R)) &&
- "Store recipes should not define any values");
+ assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)) &&
+ "Store recipes should not define any values");
return cast<LoadInst>(&R->getIngredient())->getType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9b91cb1235cda..f2f006a4481ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -91,7 +91,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
- case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
case VPWidenPHISC:
@@ -115,7 +114,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
case VPInstructionSC:
return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
- case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
return true;
@@ -211,7 +209,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPInterleaveEVLSC:
case VPInterleaveSC:
return mayWriteToMemory();
- case VPWidenStridedLoadSC:
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
case VPWidenStoreEVLSC:
@@ -3881,57 +3878,6 @@ void VPWidenLoadEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPWidenStridedLoadRecipe::execute(VPTransformState &State) {
- Type *ScalarDataTy = getLoadStoreType(&Ingredient);
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
-
- auto &Builder = State.Builder;
- Value *Addr = State.get(getAddr(), /*IsScalar*/ true);
- Value *StrideInBytes = State.get(getStride(), /*IsScalar*/ true);
- Value *Mask = nullptr;
- if (VPValue *VPMask = getMask())
- Mask = State.get(VPMask);
- else
- Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
- Value *RunTimeVF = Builder.CreateZExtOrTrunc(State.get(getVF(), VPLane(0)),
- Builder.getInt32Ty());
-
- auto *PtrTy = Addr->getType();
- auto *StrideTy = StrideInBytes->getType();
- CallInst *NewLI = Builder.CreateIntrinsic(
- Intrinsic::experimental_vp_strided_load, {DataTy, PtrTy, StrideTy},
- {Addr, StrideInBytes, Mask, RunTimeVF}, nullptr, "wide.strided.load");
- NewLI->addParamAttr(
- 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- applyMetadata(*NewLI);
- State.set(this, NewLI);
-}
-
-InstructionCost
-VPWidenStridedLoadRecipe::computeCost(ElementCount VF,
- VPCostContext &Ctx) const {
- Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
- const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
- return Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load, Ty,
- Ptr, IsMasked, Alignment, &Ingredient),
- Ctx.CostKind);
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPWidenStridedLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "WIDEN ";
- printAsOperand(O, SlotTracker);
- O << " = load ";
- getAddr()->printAsOperand(O, SlotTracker);
- O << ", stride = ";
- getStride()->printAsOperand(O, SlotTracker);
- O << ", runtimeVF = ";
- getVF()->printAsOperand(O, SlotTracker);
-}
-#endif
-
void VPWidenStoreRecipe::execute(VPTransformState &State) {
VPValue *StoredVPValue = getStoredValue();
bool CreateScatter = !isConsecutive();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b02c58d152c4a..0bccde8cffb87 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2949,14 +2949,6 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
*I, I->getDebugLoc());
}
- if (auto *StridedL = dyn_cast<VPWidenStridedLoadRecipe>(&CurRecipe))
- if (StridedL->isMasked() &&
- match(StridedL->getMask(), m_RemoveMask(HeaderMask, Mask)))
- return new VPWidenStridedLoadRecipe(
- *cast<LoadInst>(&StridedL->getIngredient()), StridedL->getAddr(),
- StridedL->getStride(), &EVL, Mask, *StridedL,
- StridedL->getDebugLoc());
-
VPValue *StoredVal;
if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal),
m_RemoveMask(HeaderMask, Mask))))
@@ -3079,17 +3071,14 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
- assert(
- all_of(
- Plan.getVF().users(),
- [&LoopRegion](VPUser *U) {
- auto *R = cast<VPRecipeBase>(U);
- return (R->getParent()->getParent() != LoopRegion) ||
- isa<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe, VPWidenStridedLoadRecipe>(
- R);
- }) &&
- "User of VF that we can't transform to EVL.");
+ assert(all_of(Plan.getVF().users(),
+ [&LoopRegion](VPUser *U) {
+ auto *R = cast<VPRecipeBase>(U);
+ return (R->getParent()->getParent() != LoopRegion) ||
+ isa<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+ VPWidenIntOrFpInductionRecipe>(R);
+ }) &&
+ "User of VF that we can't transform to EVL.");
Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
});
>From c0dad577cb8c6f303dd7c8540b6b60ca89ce4b3f Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 8 Dec 2025 23:59:31 -0800
Subject: [PATCH 22/35] inline getUniqueVariantIndex()
---
llvm/lib/Transforms/Vectorize/VPlan.h | 17 ++---------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 16 ++++++++++++----
2 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 12ffeff044ae9..4c883b64e9c29 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2149,10 +2149,6 @@ class VPHistogramRecipe : public VPRecipeBase {
class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
- bool isIndexLoopInvariant(unsigned I) const {
- return getOperand(I + 1)->isDefinedOutsideLoopRegions();
- }
-
public:
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
@@ -2182,17 +2178,8 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
return getOperand(0)->isDefinedOutsideLoopRegions();
}
- std::optional<unsigned> getUniqueVariantIndex() const {
- std::optional<unsigned> VarIdx;
- for (unsigned I = 0, E = getNumOperands() - 1; I < E; ++I) {
- if (isIndexLoopInvariant(I))
- continue;
-
- if (VarIdx)
- return std::nullopt;
- VarIdx = I;
- }
- return VarIdx;
+ bool isIndexLoopInvariant(unsigned I) const {
+ return getOperand(I + 1)->isDefinedOutsideLoopRegions();
}
/// Returns the element type for the first \p I indices of this recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0bccde8cffb87..d0641f8e90236 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6578,17 +6578,25 @@ determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
return {nullptr, nullptr, nullptr};
// Find the only one variant index.
- std::optional<unsigned> VarIndex = WidenGEP->getUniqueVariantIndex();
- if (!VarIndex)
+ unsigned VarOp = 0;
+ for (unsigned I = 1, E = WidenGEP->getNumOperands(); I < E; ++I) {
+ if (WidenGEP->isIndexLoopInvariant(I - 1))
+ continue;
+
+ if (VarOp != 0)
+ return {nullptr, nullptr, nullptr};
+ VarOp = I;
+ }
+
+ if (!VarOp)
return {nullptr, nullptr, nullptr};
- Type *ElementTy = WidenGEP->getIndexedType(*VarIndex);
+ Type *ElementTy = WidenGEP->getIndexedType(VarOp - 1);
assert(!ElementTy->isScalableTy() && !ElementTy->isVectorTy() &&
"Unexpected indexed type");
if (ElementTy->isStructTy())
return {nullptr, nullptr, nullptr};
- unsigned VarOp = *VarIndex + 1;
VPValue *IndexVPV = WidenGEP->getOperand(VarOp);
auto [Start, Stride] = matchStridedStart(IndexVPV);
if (!Start)
>From 4f715da66d4908fcfc933aff2a2a8eb867b160ea Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 9 Dec 2025 01:00:32 -0800
Subject: [PATCH 23/35] use createOverflowingOp to generate VPInstruction
recipe
---
.../Vectorize/LoopVectorizationPlanner.h | 14 ++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 19 +++++++------------
2 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a6789974e0bd6..3f583f1098840 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -434,6 +434,20 @@ class VPBuilder {
return tryInsertInstruction(new VPExpandSCEVRecipe(Expr));
}
+ VPVectorPointerRecipe *
+ createVectorPointer(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
+ GEPNoWrapFlags GEPFlags, DebugLoc DL) {
+ return tryInsertInstruction(
+ new VPVectorPointerRecipe(Ptr, SourceElementTy, Stride, GEPFlags, DL));
+ }
+
+ VPWidenMemIntrinsicRecipe *
+ createWidenMemIntrinsic(LoadInst &LI, ArrayRef<VPValue *> CallArguments,
+ const VPIRMetadata &MD, DebugLoc DL) {
+ return tryInsertInstruction(
+ new VPWidenMemIntrinsicRecipe(LI, CallArguments, MD, DL));
+ }
+
//===--------------------------------------------------------------------===//
// RAII helpers.
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d0641f8e90236..9b485a869eb97 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6559,11 +6559,10 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
WidenR->getDebugLoc());
StartR->insertBefore(WidenR);
+ VPBuilder Builder(WidenR);
unsigned InvIdx = VarIdx == 0 ? 1 : 0;
auto *StrideR =
- new VPInstruction(Opcode, {Stride, WidenR->getOperand(InvIdx)},
- VPRecipeWithIRFlags::WrapFlagsTy{false, false});
- StrideR->insertBefore(WidenR);
+ Builder.createOverflowingOp(Opcode, {Stride, WidenR->getOperand(InvIdx)});
return {StartR, StrideR};
}
@@ -6684,11 +6683,11 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
TypeInfo.inferScalarType(&Plan.getVF()), DebugLoc::getUnknown());
}
+ VPBuilder Builder(LoadR);
// Create a new vector pointer for strided access.
- auto *NewPtr = new VPVectorPointerRecipe(
+ auto *NewPtr = Builder.createVectorPointer(
BasePtr, ElementTy, StrideInElement, Ptr->getGEPNoWrapFlags(),
Ptr->getDebugLoc());
- NewPtr->insertBefore(LoadR);
const DataLayout &DL = Ingredient.getDataLayout();
TypeSize TS = DL.getTypeAllocSize(ElementTy);
@@ -6698,11 +6697,8 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
if (TypeScale != 1) {
VPValue *ScaleVPV = Plan.getConstantInt(
TypeInfo.inferScalarType(StrideInElement), TypeScale);
- auto *ScaledStride =
- new VPInstruction(Instruction::Mul, {StrideInElement, ScaleVPV},
- VPRecipeWithIRFlags::WrapFlagsTy{false, false});
- ScaledStride->insertBefore(LoadR);
- StrideInBytes = ScaledStride;
+ StrideInBytes = Builder.createOverflowingOp(
+ Instruction::Mul, {StrideInElement, ScaleVPV});
}
VPValue *Mask;
@@ -6710,10 +6706,9 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
Mask = LoadMask;
else
Mask = Plan.getTrue();
- auto *StridedLoad = new VPWidenMemIntrinsicRecipe(
+ auto *StridedLoad = Builder.createWidenMemIntrinsic(
*cast<LoadInst>(&Ingredient), {NewPtr, StrideInBytes, Mask, I32VF},
*LoadR, LoadR->getDebugLoc());
- StridedLoad->insertBefore(LoadR);
LoadR->replaceAllUsesWith(StridedLoad);
ToErase.push_back(LoadR);
>From fec90fbb3cb5bd8e2b67541864aa00fc8f7e12e0 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 9 Dec 2025 01:41:47 -0800
Subject: [PATCH 24/35] Replace VPReplicateRecipe with VPInstruction
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9b485a869eb97..77995b7826296 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6551,15 +6551,13 @@ static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
if (!Start)
return {nullptr, nullptr};
+ VPBuilder Builder(WidenR);
SmallVector<VPValue *> StartOps(WidenR->operands());
StartOps[VarIdx] = Start;
- auto *StartR = new VPReplicateRecipe(WidenR->getUnderlyingInstr(), StartOps,
- /*IsUniform*/ true, /*Mask*/ nullptr,
- /*Flags*/ *WidenR, /*Metadata*/ *WidenR,
- WidenR->getDebugLoc());
- StartR->insertBefore(WidenR);
-
- VPBuilder Builder(WidenR);
+ auto *StartR = Builder.createOverflowingOp(
+ Opcode, StartOps,
+ {WidenR->hasNoUnsignedWrap(), WidenR->hasNoSignedWrap()},
+ WidenR->getDebugLoc());
unsigned InvIdx = VarIdx == 0 ? 1 : 0;
auto *StrideR =
Builder.createOverflowingOp(Opcode, {Stride, WidenR->getOperand(InvIdx)});
>From 0df2f9d82fd1ffdd556fa6839cef7e25903e8f84 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 12 Jan 2026 01:10:31 -0800
Subject: [PATCH 25/35] Rename Offset to VFxPart
---
llvm/lib/Transforms/Vectorize/VPlan.h | 10 +++++-----
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +++++++++---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 7 +------
.../LoopVectorize/RISCV/strided-accesses.ll | 4 ++--
.../VPlan/dissolve-replicate-regions.ll | 9 ++++-----
6 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4c883b64e9c29..37d1a72c73ad2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2289,8 +2289,8 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
/// A recipe to compute the pointers for widened memory accesses of \p
/// SourceElementTy, with the \p Stride expressed in units of \p
-/// SourceElementTy. Unrolling adds an extra offset operand for unrolled parts >
-/// 0 and it produces `GEP Ptr, Offset`. The offset for unrolled part 0 is 0.
+/// SourceElementTy. Unrolling adds an extra \p VFxPart operand for unrolled
+/// parts > 0 and it produces `GEP Ptr, VFxPart * Stride`.
class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
@@ -2305,7 +2305,7 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
VPValue *getStride() const { return getOperand(1); }
- VPValue *getOffset() {
+ VPValue *getVFxPart() const {
return getNumOperands() > 2 ? getOperand(2) : nullptr;
}
@@ -2331,8 +2331,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
auto *Clone =
new VPVectorPointerRecipe(getOperand(0), SourceElementTy, getStride(),
getGEPNoWrapFlags(), getDebugLoc());
- if (auto *Off = getOffset())
- Clone->addOperand(Off);
+ if (auto *VFxPart = getVFxPart())
+ Clone->addOperand(VFxPart);
return Clone;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f2f006a4481ae..42b9fb578ade7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2778,11 +2778,17 @@ void VPVectorEndPointerRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
void VPVectorPointerRecipe::execute(VPTransformState &State) {
+ assert(getVFxPart() &&
+ "Expected prior simplification of recipe without VFxPart");
+
auto &Builder = State.Builder;
- assert(getOffset() &&
- "Expected prior simplification of recipe without offset");
Value *Ptr = State.get(getOperand(0), VPLane(0));
- Value *Offset = State.get(getOffset(), true);
+ Value *Offset = State.get(getVFxPart(), true);
+ if (!match(getStride(), m_One())) {
+ Value *Stride = Builder.CreateZExtOrTrunc(State.get(getStride(), true),
+ Offset->getType());
+ Offset = Builder.CreateMul(Offset, Stride);
+ }
Value *ResultPtr = Builder.CreateGEP(getSourceElementType(), Ptr, Offset, "",
getGEPNoWrapFlags());
State.set(this, ResultPtr, /*IsScalar*/ true);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 77995b7826296..c7fcdfeeffd47 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1769,7 +1769,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
// Simplify unrolled VectorPointer without offset, or with zero offset, to
// just the pointer operand.
if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Def))
- if (!VPR->getOffset() || match(VPR->getOffset(), m_ZeroInt()))
+ if (!VPR->getVFxPart() || match(VPR->getVFxPart(), m_ZeroInt()))
return VPR->replaceAllUsesWith(VPR->getOperand(0));
// VPScalarIVSteps after unrolling can be replaced by their start value, if
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 4e3af417a1e98..0e0bda43fd0be 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -352,14 +352,9 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
VPValue *VFxPart = Builder.createOverflowingOp(
Instruction::Mul, {VF, Plan.getConstantInt(IndexTy, Part)},
{true, true});
- VPValue *Stride = Builder.createScalarZExtOrTrunc(
- VPR->getStride(), IndexTy, TypeInfo.inferScalarType(VPR->getStride()),
- DebugLoc::getUnknown());
- VPValue *Offset =
- Builder.createOverflowingOp(Instruction::Mul, {VFxPart, Stride});
Copy->setOperand(0, VPR->getOperand(0));
Copy->setOperand(1, VPR->getOperand(1));
- Copy->addOperand(Offset);
+ Copy->addOperand(VFxPart);
continue;
}
if (auto *Red = dyn_cast<VPReductionRecipe>(&R)) {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 3925aa548831f..2ee5181baf6cc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -68,7 +68,7 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP8]]
; CHECK-UF2-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP9]]
; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
-; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 3
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP3]], 8
; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP20]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
@@ -179,7 +179,7 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP17]]
; CHECK-UF2-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
; CHECK-UF2-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[STEP_ADD]]
-; CHECK-UF2-NEXT: [[TMP19:%.*]] = shl i64 [[TMP3]], 6
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP3]], 64
; CHECK-UF2-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index ced2cebb11e04..9f0421f55e619 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -160,11 +160,10 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%25>/vp<[[VP5]]>
; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP7]]>
; VECTOR-NEXT: EMIT vp<[[VP26:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: EMIT vp<[[VP27:%[0-9]+]]> = mul vp<[[VP26]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP28:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>
-; VECTOR-NEXT: vp<[[VP29:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>, vp<[[VP27]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP28]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP29]]>, ir<%pred.val>.1
+; VECTOR-NEXT: vp<[[VP27:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP28:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>, vp<[[VP26]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP27]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP28]]>, ir<%pred.val>.1
; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
>From 3a009c6b5d277615eb80b73b2dcd4a3dec483880 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 9 Feb 2026 17:20:55 -0800
Subject: [PATCH 26/35] Use VPlan-based SCEV analysis
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 23 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 150 ++++--------------
.../Transforms/Vectorize/VPlanTransforms.h | 5 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 7 +-
.../RISCV/masked_gather_scatter.ll | 24 +--
.../LoopVectorize/RISCV/pointer-induction.ll | 8 +-
.../LoopVectorize/RISCV/reg-usage-prune-vf.ll | 104 ++++--------
.../runtime-check-dependent-on-stride.ll | 17 +-
.../LoopVectorize/RISCV/strided-accesses.ll | 62 ++++----
.../RISCV/tail-folding-gather-scatter.ll | 6 +-
.../RISCV/tail-folding-interleave.ll | 45 ++++--
.../truncate-to-minimal-bitwidth-cost.ll | 81 +++++++++-
13 files changed, 256 insertions(+), 280 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c27b441fae398..5785270694cad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7098,8 +7098,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
- RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, CostCtx,
- Range);
+ RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
+ *OrigLoop, CostCtx, Range);
// Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
if (Range.Start.isScalar())
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 37d1a72c73ad2..75e629d5dfdac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2149,6 +2149,14 @@ class VPHistogramRecipe : public VPRecipeBase {
class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
+ bool isPointerLoopInvariant() const {
+ return getOperand(0)->isDefinedOutsideLoopRegions();
+ }
+
+ bool isIndexLoopInvariant(unsigned I) const {
+ return getOperand(I + 1)->isDefinedOutsideLoopRegions();
+ }
+
public:
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
@@ -2174,21 +2182,6 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
/// This recipe generates a GEP instruction.
unsigned getOpcode() const { return Instruction::GetElementPtr; }
- bool isPointerLoopInvariant() const {
- return getOperand(0)->isDefinedOutsideLoopRegions();
- }
-
- bool isIndexLoopInvariant(unsigned I) const {
- return getOperand(I + 1)->isDefinedOutsideLoopRegions();
- }
-
- /// Returns the element type for the first \p I indices of this recipe.
- Type *getIndexedType(unsigned I) const {
- auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
- SmallVector<Value *, 4> Ops(GEP->idx_begin(), GEP->idx_begin() + I);
- return GetElementPtrInst::getIndexedType(SourceElementTy, Ops);
- }
-
/// Generate the gep nodes.
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c7fcdfeeffd47..65559ac90cd96 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2939,8 +2939,7 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
if (match(&CurRecipe, m_Intrinsic<Intrinsic::experimental_vp_strided_load>(
m_VPValue(Addr), m_VPValue(Stride),
m_RemoveMask(HeaderMask, Mask),
- m_VPInstruction<Instruction::Trunc>(
- m_Specific(&Plan->getVF()))))) {
+ m_TruncOrSelf(m_Specific(&Plan->getVF()))))) {
auto *I = cast<VPWidenIntrinsicRecipe>(&CurRecipe);
if (!Mask)
Mask = Plan->getTrue();
@@ -3076,7 +3075,8 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
auto *R = cast<VPRecipeBase>(U);
return (R->getParent()->getParent() != LoopRegion) ||
isa<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe>(R);
+ VPWidenIntOrFpInductionRecipe,
+ VPWidenMemIntrinsicRecipe>(R);
}) &&
"User of VF that we can't transform to EVL.");
Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
@@ -6525,103 +6525,19 @@ void VPlanTransforms::makeMemOpWideningDecisions(
}
}
-static std::pair<VPValue *, VPValue *> matchStridedStart(VPValue *CurIndex) {
- // TODO: Support VPWidenPointerInductionRecipe.
- if (auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(CurIndex))
- return {WidenIV, WidenIV->getStepValue()};
-
- auto *WidenR = dyn_cast<VPWidenRecipe>(CurIndex);
- if (!WidenR || !WidenR->getUnderlyingInstr())
- return {nullptr, nullptr};
-
- unsigned Opcode = WidenR->getOpcode();
- // TODO: Support Instruction::Add and Instruction::Or.
- if (Opcode != Instruction::Shl && Opcode != Instruction::Mul)
- return {nullptr, nullptr};
-
- // Match the pattern binop(variant, uniform), or binop(uniform, variant) if
- // the binary operator is commutative.
- bool IsLHSUniform = vputils::isSingleScalar(WidenR->getOperand(0));
- if (IsLHSUniform == vputils::isSingleScalar(WidenR->getOperand(1)) ||
- (IsLHSUniform && !Instruction::isCommutative(Opcode)))
- return {nullptr, nullptr};
- unsigned VarIdx = IsLHSUniform ? 1 : 0;
-
- auto [Start, Stride] = matchStridedStart(WidenR->getOperand(VarIdx));
- if (!Start)
- return {nullptr, nullptr};
-
- VPBuilder Builder(WidenR);
- SmallVector<VPValue *> StartOps(WidenR->operands());
- StartOps[VarIdx] = Start;
- auto *StartR = Builder.createOverflowingOp(
- Opcode, StartOps,
- {WidenR->hasNoUnsignedWrap(), WidenR->hasNoSignedWrap()},
- WidenR->getDebugLoc());
- unsigned InvIdx = VarIdx == 0 ? 1 : 0;
- auto *StrideR =
- Builder.createOverflowingOp(Opcode, {Stride, WidenR->getOperand(InvIdx)});
- return {StartR, StrideR};
-}
-
-/// Checks if the given VPWidenGEPRecipe \p WidenGEP represents a strided
-/// access. If so, it creates recipes representing the base pointer and stride
-/// in element units, and returns a tuple of {base pointer, stride, element
-/// type}. Otherwise, returns a tuple where all elements are nullptr.
-static std::tuple<VPValue *, VPValue *, Type *>
-determineBaseAndStride(VPWidenGEPRecipe *WidenGEP) {
- // TODO: Check if the base pointer is strided.
- if (!WidenGEP->isPointerLoopInvariant())
- return {nullptr, nullptr, nullptr};
-
- // Find the only one variant index.
- unsigned VarOp = 0;
- for (unsigned I = 1, E = WidenGEP->getNumOperands(); I < E; ++I) {
- if (WidenGEP->isIndexLoopInvariant(I - 1))
- continue;
-
- if (VarOp != 0)
- return {nullptr, nullptr, nullptr};
- VarOp = I;
- }
-
- if (!VarOp)
- return {nullptr, nullptr, nullptr};
-
- Type *ElementTy = WidenGEP->getIndexedType(VarOp - 1);
- assert(!ElementTy->isScalableTy() && !ElementTy->isVectorTy() &&
- "Unexpected indexed type");
- if (ElementTy->isStructTy())
- return {nullptr, nullptr, nullptr};
-
- VPValue *IndexVPV = WidenGEP->getOperand(VarOp);
- auto [Start, Stride] = matchStridedStart(IndexVPV);
- if (!Start)
- return {nullptr, nullptr, nullptr};
-
- SmallVector<VPValue *> Ops(WidenGEP->operands());
- Ops[VarOp] = Start;
- auto *BasePtr = new VPReplicateRecipe(
- WidenGEP->getUnderlyingInstr(), Ops,
- /*IsUniform*/ true, /*Mask*/ nullptr, /*Flags*/ *WidenGEP,
- /*Metadata*/ {}, WidenGEP->getDebugLoc());
- BasePtr->insertBefore(WidenGEP);
-
- return {BasePtr, Stride, ElementTy};
-}
-
-void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
+void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
+ PredicatedScalarEvolution &PSE,
+ Loop &L, VPCostContext &Ctx,
VFRange &Range) {
if (Plan.hasScalarVFOnly())
return;
VPTypeAnalysis TypeInfo(Plan);
- DenseMap<VPWidenGEPRecipe *, std::tuple<VPValue *, VPValue *, Type *>>
- StrideCache;
+ VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
SmallVector<VPWidenMemoryRecipe *> ToErase;
VPValue *I32VF = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ vp_depth_first_shallow(VectorLoop->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
auto *LoadR = dyn_cast<VPWidenLoadRecipe>(&R);
// TODO: Support strided store.
@@ -6657,21 +6573,13 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
Range))
continue;
- // Try to get base and stride here.
- VPValue *BasePtr, *StrideInElement;
- Type *ElementTy;
- auto It = StrideCache.find(Ptr);
- if (It != StrideCache.end())
- std::tie(BasePtr, StrideInElement, ElementTy) = It->second;
- else
- std::tie(BasePtr, StrideInElement, ElementTy) = StrideCache[Ptr] =
- determineBaseAndStride(Ptr);
-
- // Skip if the memory access is not a strided access.
- if (!BasePtr)
+ const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
+ const SCEV *Start;
+ const APInt *Step;
+ // TODO: Support loop invariant stride.
+ if (!match(PtrSCEV,
+ m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step))))
continue;
- assert(StrideInElement && ElementTy &&
- "Can not get stride information for a strided access");
// Add VF of i32 version for EVL.
if (!I32VF) {
@@ -6682,22 +6590,24 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
}
VPBuilder Builder(LoadR);
+ // Create the base pointer of strided access.
+ VPValue *StartVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Start);
+ VPValue *StrideInBytes =
+ Plan.getConstantInt(VectorLoop->getCanonicalIVType(),
+ Step->getSExtValue(), /*IsSigned=*/true);
+ auto *AddRecPtr = cast<SCEVAddRecExpr>(PtrSCEV);
+ auto *Offset = Builder.createOverflowingOp(
+ Instruction::Mul, {VectorLoop->getCanonicalIV(), StrideInBytes},
+ {AddRecPtr->hasNoUnsignedWrap(), AddRecPtr->hasNoSignedWrap()});
+ auto *BasePtr = Builder.createNoWrapPtrAdd(
+ StartVPV, Offset,
+ AddRecPtr->hasNoUnsignedWrap() ? GEPNoWrapFlags::noUnsignedWrap()
+ : GEPNoWrapFlags::none());
+
// Create a new vector pointer for strided access.
auto *NewPtr = Builder.createVectorPointer(
- BasePtr, ElementTy, StrideInElement, Ptr->getGEPNoWrapFlags(),
- Ptr->getDebugLoc());
-
- const DataLayout &DL = Ingredient.getDataLayout();
- TypeSize TS = DL.getTypeAllocSize(ElementTy);
- unsigned TypeScale = TS.getFixedValue();
- VPValue *StrideInBytes = StrideInElement;
- // Scale the stride by the size of the indexed type.
- if (TypeScale != 1) {
- VPValue *ScaleVPV = Plan.getConstantInt(
- TypeInfo.inferScalarType(StrideInElement), TypeScale);
- StrideInBytes = Builder.createOverflowingOp(
- Instruction::Mul, {StrideInElement, ScaleVPV});
- }
+ BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes,
+ Ptr->getGEPNoWrapFlags(), Ptr->getDebugLoc());
VPValue *Mask;
if (VPValue *LoadMask = LoadR->getMask())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b67b22d56f364..cacd7fea3d14d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -330,8 +330,9 @@ struct VPlanTransforms {
/// Transform widen memory recipes into strided access recipes when legal
/// and profitable. Clamps \p Range to maintain consistency with widen
/// decisions of \p Plan, and uses \p Ctx to evaluate the cost.
- static void convertToStridedAccesses(VPlan &Plan, VPCostContext &Ctx,
- VFRange &Range);
+ static void convertToStridedAccesses(VPlan &Plan,
+ PredicatedScalarEvolution &PSE, Loop &L,
+ VPCostContext &Ctx, VFRange &Range);
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 791130f004e0e..afe900497afe9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -83,6 +83,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 16 x i32> [[TMP0]], splat (i32 4)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 252, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
@@ -90,10 +91,12 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i32> [[VEC_IND]] to <vscale x 16 x i64>
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 16 x i64> [[TMP9]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = shl i32 [[EVL_BASED_IV]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP12]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i32(ptr align 1 [[TMP13]], i32 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 16 x i64> [[TMP9]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP2]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 82c85e6a5a801..3b20de992d6c2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -57,13 +57,13 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV32-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 4
-; RV32-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP16]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP26]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; RV32-NEXT: [[TMP12:%.*]] = shl i64 [[INDEX]], 6
+; RV32-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TRIGGER]], i64 [[TMP12]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP13]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; RV32-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV32-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP16]], 1
-; RV32-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP28]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP29]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]]
+; RV32-NEXT: [[TMP16:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
+; RV32-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP16]]
+; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP20]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]]
; RV32-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV32-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV32-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
@@ -127,13 +127,13 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP8]], 4
; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP4]], i64 0
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; RV64-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 4
-; RV64-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP9]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP25]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT: [[TMP5:%.*]] = shl nuw i64 [[INDEX]], 6
+; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[TRIGGER]], i64 [[TMP5]]
+; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP6]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
-; RV64-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP9]], 1
-; RV64-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP26]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP15]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT: [[TMP9:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
+; RV64-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP9]]
+; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP11]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
; RV64-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
; RV64-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
; RV64-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 786ef735fc7ad..bd3ccea350324 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -71,6 +71,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]]
; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
@@ -91,19 +92,22 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP3]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 12)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 4
-; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[EVL_BASED_IV]], 12
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP15]], i64 12, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP26]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP27]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
index 2ae485fc758f2..19435c1026175 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
@@ -7,50 +7,32 @@ define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) {
; CHECK-LABEL: define void @f(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P0]], i64 -1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 2)
-; CHECK-NEXT: [[TMP3:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 3)
-; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 4)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 4 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP7]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = mul i64 3, [[TMP7]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = sub <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP13]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = sub <vscale x 4 x i64> [[VEC_IND3]], splat (i64 1)
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP15]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP17:%.*]] = sub <vscale x 4 x i64> [[VEC_IND4]], splat (i64 1)
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP17]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[EVL_BASED_IV]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP2]], i64 2, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP7]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP5]], i64 3, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP7]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP7]], 3
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP19]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i8 0
; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP6]], 3
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i8> @llvm.vector.interleave3.nxv12i8(<vscale x 4 x i8> [[WIDE_MASKED_GATHER]], <vscale x 4 x i8> [[WIDE_MASKED_GATHER9]], <vscale x 4 x i8> [[WIDE_MASKED_GATHER10]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv12i8.p0(<vscale x 12 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 12 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[EVL_BASED_IV]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
-; CHECK-NEXT: [[VEC_IND_NEXT9]] = add <vscale x 4 x i64> [[VEC_IND3]], [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[VEC_IND_NEXT10]] = add <vscale x 4 x i64> [[VEC_IND4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP9]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv48i8.p0(<vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 48 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[TMP7]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -61,50 +43,32 @@ define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) {
; NO-REG-PRESSURE-CHECK-LABEL: define void @f(
; NO-REG-PRESSURE-CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
; NO-REG-PRESSURE-CHECK-NEXT: [[ENTRY:.*:]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P0]], i64 -1
; NO-REG-PRESSURE-CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; NO-REG-PRESSURE-CHECK: [[VECTOR_PH]]:
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 2)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP3:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 3)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 4)
; NO-REG-PRESSURE-CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-REG-PRESSURE-CHECK: [[VECTOR_BODY]]:
-; NO-REG-PRESSURE-CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND3:%.*]] = phi <vscale x 8 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 8 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], %[[VECTOR_BODY]] ]
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-REG-PRESSURE-CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP7]], 2
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP10]], i64 0
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP11:%.*]] = mul i64 3, [[TMP7]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP11]], i64 0
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP8]], i64 0
-; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP9:%.*]] = sub <vscale x 8 x i64> [[VEC_IND]], splat (i64 1)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP9]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP14]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP15:%.*]] = sub <vscale x 8 x i64> [[VEC_IND3]], splat (i64 1)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP15]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP16]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP17:%.*]] = sub <vscale x 8 x i64> [[VEC_IND4]], splat (i64 1)
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP17]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP18]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[EVL_BASED_IV]], 3
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP8]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP2]], i64 2, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP7]], 3
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP5]], i64 3, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP7]], 2
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP7]], 3
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP19]]
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i8 0
; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP6]], 3
-; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 24 x i8> @llvm.vector.interleave3.nxv24i8(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x i8> [[WIDE_MASKED_GATHER9]], <vscale x 8 x i8> [[WIDE_MASKED_GATHER10]])
-; NO-REG-PRESSURE-CHECK-NEXT: call void @llvm.vp.store.nxv24i8.p0(<vscale x 24 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 24 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[EVL_BASED_IV]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT9]] = add <vscale x 8 x i64> [[VEC_IND3]], [[BROADCAST_SPLAT4]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT10]] = add <vscale x 8 x i64> [[VEC_IND4]], [[BROADCAST_SPLAT]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP9]])
+; NO-REG-PRESSURE-CHECK-NEXT: call void @llvm.vp.store.nxv48i8.p0(<vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 48 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; NO-REG-PRESSURE-CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[TMP7]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-REG-PRESSURE-CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; NO-REG-PRESSURE-CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
index 8a6108a0248ba..1a665c21d1e55 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
@@ -81,24 +81,31 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_PH]]:
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP28:%.*]] = shl i64 [[STRIDE]], 3
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[STRIDE]], i64 0
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-UNIT-STRIDE-MV-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 2 x i64> splat (i64 1), [[TMP19]]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
+; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP26]], i64 0
+; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT12]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; NO-UNIT-STRIDE-MV-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul i64 [[OFFSET_IDX]], [[STRIDE]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP24:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[OFFSET_IDX]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[P_STRIDED]], i64 [[TMP34]]
+; NO-UNIT-STRIDE-MV-NEXT: [[TMP27:%.*]] = getelementptr i64, ptr [[P_STRIDED]], <vscale x 2 x i64> [[TMP24]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_LOAD19:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP30]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META0:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP29]], i64 [[TMP28]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
+; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP27]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
+; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT13]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; NO-UNIT-STRIDE-MV: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 2ee5181baf6cc..6e22a64b69aa4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -21,10 +21,10 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP14]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 5
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP9]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -62,14 +62,14 @@ define void @single_constant_stride_int_scaled(ptr %p) {
; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-UF2-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; CHECK-UF2-NEXT: [[TMP9:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-UF2-NEXT: [[TMP10:%.*]] = shl nuw nsw <vscale x 4 x i64> [[STEP_ADD]], splat (i64 3)
-; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP8]]
; CHECK-UF2-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP9]]
; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
-; CHECK-UF2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP3]], 8
-; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = shl i64 [[INDEX]], 5
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP17]]
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP3]], 32
+; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP18]], i64 [[TMP19]]
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP20]], i64 32, <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; CHECK-UF2-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
@@ -134,9 +134,9 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP11]], 6
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 6
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP15]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
@@ -175,12 +175,12 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[TMP6]]
-; CHECK-UF2-NEXT: [[TMP17:%.*]] = shl i64 [[INDEX]], 6
-; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP17]]
; CHECK-UF2-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
; CHECK-UF2-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[STEP_ADD]]
-; CHECK-UF2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP3]], 64
-; CHECK-UF2-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP18]], i64 [[TMP19]]
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 8
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP16]]
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = mul i64 [[TMP3]], 256
+; CHECK-UF2-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP18]], i64 [[TMP17]]
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
; CHECK-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 256, <vscale x 4 x i1> splat (i1 true), i32 [[TMP15]])
; CHECK-UF2-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
@@ -797,27 +797,23 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; STRIDED-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; STRIDED: [[VECTOR_PH]]:
-; STRIDED-NEXT: [[TMP46:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED: [[VECTOR_BODY]]:
-; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP12]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP43:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; STRIDED-NEXT: [[TMP44:%.*]] = zext i32 [[TMP43]] to i64
; STRIDED-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP44]], i64 0
; STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; STRIDED-NEXT: [[TMP49:%.*]] = mul nuw nsw i64 [[INDEX]], [[STRIDE]]
; STRIDED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT1]]
-; STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP49]]
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP50]], i64 [[TMP46]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META5:![0-9]+]]
+; STRIDED-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP18]]
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP42]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META5:![0-9]+]]
; STRIDED-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP18]]
; STRIDED-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META8:![0-9]+]], !noalias [[META5]]
-; STRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP44]], [[INDEX]]
; STRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP44]]
; STRIDED-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; STRIDED-NEXT: [[TMP41:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -903,8 +899,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 1
; STRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP30]]
; STRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; STRIDED-UF2-NEXT: [[TMP44:%.*]] = trunc i64 [[TMP29]] to i32
-; STRIDED-UF2-NEXT: [[TMP32:%.*]] = shl i64 [[STRIDE]], 2
; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT10]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-UF2-NEXT: [[TMP31:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -913,14 +907,12 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-UF2-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP31]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-UF2-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; STRIDED-UF2-NEXT: [[TMP35:%.*]] = mul nuw nsw i64 [[INDEX]], [[STRIDE]]
; STRIDED-UF2-NEXT: [[TMP33:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT11]]
; STRIDED-UF2-NEXT: [[TMP34:%.*]] = mul nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT11]]
-; STRIDED-UF2-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP35]]
-; STRIDED-UF2-NEXT: [[TMP42:%.*]] = mul i64 [[TMP29]], [[STRIDE]]
-; STRIDED-UF2-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP36]], i64 [[TMP42]]
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP36]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP44]]), !alias.scope [[META8:![0-9]+]]
-; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP43]], i64 [[TMP32]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP44]]), !alias.scope [[META8]]
+; STRIDED-UF2-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP33]]
+; STRIDED-UF2-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP34]]
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP36]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META8:![0-9]+]]
+; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP35]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META8]]
; STRIDED-UF2-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER12]], splat (i32 1)
; STRIDED-UF2-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP33]]
@@ -1335,7 +1327,8 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; NOSTRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
+; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[EVL_BASED_IV]], 2
+; NOSTRIDED-NEXT: [[TMP9:%.*]] = getelementptr nuw i8, ptr [[IN]], i64 [[TMP1]]
; NOSTRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; NOSTRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
@@ -1366,8 +1359,10 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; NOSTRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED-UF2: [[VECTOR_BODY]]:
; NOSTRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
-; NOSTRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; NOSTRIDED-UF2-NEXT: [[TMP10:%.*]] = shl nuw i64 [[INDEX]], 2
+; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr nuw i8, ptr [[IN]], i64 [[TMP10]]
+; NOSTRIDED-UF2-NEXT: [[TMP8:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP8]]
; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
@@ -1405,7 +1400,8 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; STRIDED-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[EVL_BASED_IV]]
+; STRIDED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[EVL_BASED_IV]], 2
+; STRIDED-NEXT: [[TMP9:%.*]] = getelementptr nuw i8, ptr [[IN]], i64 [[TMP1]]
; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP9]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
; STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[EVL_BASED_IV]]
; STRIDED-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
@@ -1436,8 +1432,10 @@ define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
; STRIDED-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; STRIDED-UF2: [[VECTOR_BODY]]:
; STRIDED-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; STRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[IN]], i64 [[INDEX]]
-; STRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; STRIDED-UF2-NEXT: [[TMP10:%.*]] = shl nuw i64 [[INDEX]], 2
+; STRIDED-UF2-NEXT: [[TMP5:%.*]] = getelementptr nuw i8, ptr [[IN]], i64 [[TMP10]]
+; STRIDED-UF2-NEXT: [[TMP8:%.*]] = mul i64 [[TMP3]], 4
+; STRIDED-UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP8]]
; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; STRIDED-UF2-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; STRIDED-UF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[OUT]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
index 15b2cb0d0681c..5b177f076a7fa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
@@ -17,7 +17,8 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
; IF-EVL-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[FOR_BODY1]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[FOR_BODY1]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; IF-EVL-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV1]]
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[INDVARS_IV1]], 2
+; IF-EVL-NEXT: [[ARRAYIDX3:%.*]] = getelementptr nuw i8, ptr [[INDEX:%.*]], i64 [[TMP1]]
; IF-EVL-NEXT: [[WIDE_STRIDED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[ARRAYIDX3]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; IF-EVL-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x float> @llvm.vp.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP2]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
@@ -48,7 +49,8 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
; NO-VP-NEXT: br label [[FOR_BODY1:%.*]]
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
-; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV1]]
+; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[INDVARS_IV1]], 2
+; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr nuw i8, ptr [[INDEX:%.*]], i64 [[TMP10]]
; NO-VP-NEXT: [[WIDE_STRIDED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[ARRAYIDX3]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_STRIDED_LOAD]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP6]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x float> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index e625d0687e4bc..441e9b69bf663 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -342,6 +342,8 @@ exit:
define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; IF-EVL-LABEL: @load_factor_4_with_tail_gap(
; IF-EVL-NEXT: entry:
+; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4
+; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 8
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -350,13 +352,14 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[INDEX]], 4
+; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP1]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 1
+; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP1]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
-; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 2
+; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP1]], i64 [[TMP1]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
@@ -554,6 +557,14 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
; IF-EVL-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
+; IF-EVL-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 4
+; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP2]]
+; IF-EVL-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
+; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 8
+; IF-EVL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP2]], 12
+; IF-EVL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -562,17 +573,17 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP21:%.*]] = sub i64 [[N]], [[INDEX]]
-; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP21]], i32 0
+; IF-EVL-NEXT: [[TMP9:%.*]] = mul i64 [[INDEX]], -16
+; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP9]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 1
+; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP9]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]
-; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 2
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP9]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
-; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP21]], i32 3
+; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP9]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]
; IF-EVL-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])
@@ -592,6 +603,14 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
; NO-VP-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
+; NO-VP-NEXT: [[TMP7:%.*]] = shl i64 [[N]], 4
+; NO-VP-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP7]]
+; NO-VP-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP7]], 4
+; NO-VP-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; NO-VP-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[TMP7]], 8
+; NO-VP-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP15]]
+; NO-VP-NEXT: [[TMP23:%.*]] = add nuw nsw i64 [[TMP7]], 12
+; NO-VP-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP23]]
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
@@ -607,17 +626,17 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[INDEX]]
-; NO-VP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[TMP7]], i32 0
+; NO-VP-NEXT: [[TMP26:%.*]] = mul i64 [[INDEX]], -16
+; NO-VP-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP26]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 1
+; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP26]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]
-; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 2
+; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP26]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]
-; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP7]], i32 3
+; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP26]]
; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; NO-VP-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index f4c7c6f6fba1b..a984f776c31a5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -250,10 +250,75 @@ exit:
define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK-LABEL: define void @test_minbws_for_trunc(
; CHECK-SAME: i32 [[N:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 4, i16 255)
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP0:%.*]] = add i16 4, [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i16 [[TMP0]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP1]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 255)
+; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[P2]], i64 1021
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[P2]], i64 8168
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT8]]
+; CHECK-NEXT: [[BOUND09:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND110:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP4]]
+; CHECK-NEXT: [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
+; CHECK-NEXT: [[CONFLICT_RDX12:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT11]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i16> @llvm.stepvector.nxv2i16()
+; CHECK-NEXT: [[TMP10:%.*]] = mul <vscale x 2 x i16> [[TMP9]], splat (i16 4)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i16> [ [[TMP10]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 256, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i16
+; CHECK-NEXT: [[TMP13:%.*]] = shl i16 [[TMP12]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[TMP13]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = sext <vscale x 2 x i16> [[VEC_IND]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP17:%.*]] = shl i32 [[EVL_BASED_IV]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P1]], i32 [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i32(ptr align 4 [[TMP18]], i32 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP20:%.*]] = trunc <vscale x 2 x i32> [[TMP19]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [1 x [1 x i16]], ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i16.nxv2p0(<vscale x 2 x i16> [[TMP20]], <vscale x 2 x ptr> align 2 [[TMP21]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT: [[TMP22:%.*]] = trunc <vscale x 2 x i32> [[TMP19]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP22]], <vscale x 2 x ptr> align 1 [[TMP23]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [1 x i64], ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x ptr> align 8 [[TMP24]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META13]]
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP11]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i16> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[IV_EXT:%.*]] = sext i16 [[IV]] to i64
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i64 [[IV_EXT]]
; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[GEP1]], align 4
@@ -268,7 +333,7 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 4
; CHECK-NEXT: [[IV_NEXT_EXT:%.*]] = sext i16 [[IV_NEXT]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[IV_NEXT_EXT]], 1024
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -307,4 +372,14 @@ attributes #1 = { "target-features"="+64bit,+v" }
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META8]]}
+; CHECK: [[META12]] = !{[[META10]]}
+; CHECK: [[META13]] = !{[[META11]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
;.
>From fd39592a775bda055c9a10790e9d3bc62551aa53 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 9 Feb 2026 18:39:01 -0800
Subject: [PATCH 27/35] updated code by comment
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 65559ac90cd96..1e51d07b19974 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6605,14 +6605,12 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
: GEPNoWrapFlags::none());
// Create a new vector pointer for strided access.
- auto *NewPtr = Builder.createVectorPointer(
+ VPValue *NewPtr = Builder.createVectorPointer(
BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes,
Ptr->getGEPNoWrapFlags(), Ptr->getDebugLoc());
- VPValue *Mask;
- if (VPValue *LoadMask = LoadR->getMask())
- Mask = LoadMask;
- else
+ VPValue *Mask = LoadR->getMask();
+ if (!Mask)
Mask = Plan.getTrue();
auto *StridedLoad = Builder.createWidenMemIntrinsic(
*cast<LoadInst>(&Ingredient), {NewPtr, StrideInBytes, Mask, I32VF},
>From 5ca0fd9e07ded227a2b451515092dadc2786327d Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 9 Feb 2026 20:31:20 -0800
Subject: [PATCH 28/35] update allowed VF users function
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1e51d07b19974..843a80a6c4b97 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3071,12 +3071,15 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
assert(all_of(Plan.getVF().users(),
- [&LoopRegion](VPUser *U) {
- auto *R = cast<VPRecipeBase>(U);
- return (R->getParent()->getParent() != LoopRegion) ||
- isa<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe,
- VPWidenMemIntrinsicRecipe>(R);
+ [&Plan](VPUser *U) {
+ auto IsAllowedUser =
+ IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+ VPWidenIntOrFpInductionRecipe,
+ VPWidenMemIntrinsicRecipe>;
+ if (match(U, m_Trunc(m_Specific(&Plan.getVF()))))
+ return all_of(cast<VPSingleDefRecipe>(U)->users(),
+ IsAllowedUser);
+ return IsAllowedUser(U);
}) &&
"User of VF that we can't transform to EVL.");
Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
>From 71d625769d9f11e8c1caef441e9e58f286b9868f Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 11 Mar 2026 01:59:43 -0700
Subject: [PATCH 29/35] Regress the function until we can clean up
VPExpandSCEVRecipe
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +
.../LoopVectorize/RISCV/dead-ops-cost.ll | 10 +-
.../LoopVectorize/RISCV/pointer-induction.ll | 8 +-
.../LoopVectorize/RISCV/reg-usage-prune-vf.ll | 104 ++++++++++++------
.../RISCV/tail-folding-interleave.ll | 94 ++++++++--------
.../truncate-to-minimal-bitwidth-cost.ll | 48 ++++----
6 files changed, 152 insertions(+), 116 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 843a80a6c4b97..d9209a3aa2360 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6584,6 +6584,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step))))
continue;
+ // TODO: Support the base pointer that requires SCEV expander.
+ if (!isa<SCEVUnknown, SCEVConstant>(Start))
+ continue;
+
// Add VF of i32 version for EVL.
if (!I32VF) {
VPBuilder Builder(Plan.getVectorPreheader());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index afe900497afe9..c9e5e2d2a6faa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -83,7 +83,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 16 x i32> [[TMP0]], splat (i32 4)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 252, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
@@ -91,12 +91,12 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i32> [[VEC_IND]] to <vscale x 16 x i64>
-; CHECK-NEXT: [[TMP12:%.*]] = shl i32 [[EVL_BASED_IV]], 2
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP12]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i32(ptr align 1 [[TMP13]], i32 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i32 [[TMP5]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i32(ptr align 1 [[TMP6]], i32 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 16 x i64> [[TMP9]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP2]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP2]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index bd3ccea350324..786ef735fc7ad 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -71,7 +71,6 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]]
; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
@@ -92,22 +91,19 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP3]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 12)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[EVL_BASED_IV]], 12
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP15]], i64 12, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 4
+; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP26]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP27]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
index 19435c1026175..2ae485fc758f2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
@@ -7,32 +7,50 @@ define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) {
; CHECK-LABEL: define void @f(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P0]], i64 -1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 2)
+; CHECK-NEXT: [[TMP3:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 3)
+; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[TMP0]], splat (i64 4)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 4 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP7]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = mul i64 3, [[TMP7]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP2]], i64 2, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP7]], 3
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP5]], i64 3, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP7]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP7]], 3
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = sub <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP13]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP15:%.*]] = sub <vscale x 4 x i64> [[VEC_IND3]], splat (i64 1)
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP15]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP17:%.*]] = sub <vscale x 4 x i64> [[VEC_IND4]], splat (i64 1)
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 4 x i64> [[TMP17]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[EVL_BASED_IV]], 3
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP19]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i8 0
; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP6]], 3
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP9]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv48i8.p0(<vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 48 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
-; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[TMP7]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i8> @llvm.vector.interleave3.nxv12i8(<vscale x 4 x i8> [[WIDE_MASKED_GATHER]], <vscale x 4 x i8> [[WIDE_MASKED_GATHER9]], <vscale x 4 x i8> [[WIDE_MASKED_GATHER10]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv12i8.p0(<vscale x 12 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 12 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[VEC_IND_NEXT9]] = add <vscale x 4 x i64> [[VEC_IND3]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT10]] = add <vscale x 4 x i64> [[VEC_IND4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -43,32 +61,50 @@ define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) {
; NO-REG-PRESSURE-CHECK-LABEL: define void @f(
; NO-REG-PRESSURE-CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
; NO-REG-PRESSURE-CHECK-NEXT: [[ENTRY:.*:]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P0]], i64 -1
; NO-REG-PRESSURE-CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; NO-REG-PRESSURE-CHECK: [[VECTOR_PH]]:
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 2)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP3:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 3)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 8 x i64> [[TMP0]], splat (i64 4)
; NO-REG-PRESSURE-CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-REG-PRESSURE-CHECK: [[VECTOR_BODY]]:
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-REG-PRESSURE-CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND3:%.*]] = phi <vscale x 8 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 8 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], %[[VECTOR_BODY]] ]
; NO-REG-PRESSURE-CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP7]], 2
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP10]], i64 0
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP11:%.*]] = mul i64 3, [[TMP7]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP11]], i64 0
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP8]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP2]], i64 2, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP7]], 3
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP5]], i64 3, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP7]], 2
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP7]], 3
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP8]], i64 0
+; NO-REG-PRESSURE-CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP9:%.*]] = sub <vscale x 8 x i64> [[VEC_IND]], splat (i64 1)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP9]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP14]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP15:%.*]] = sub <vscale x 8 x i64> [[VEC_IND3]], splat (i64 1)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP15]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP16]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP17:%.*]] = sub <vscale x 8 x i64> [[VEC_IND4]], splat (i64 1)
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], <vscale x 8 x i64> [[TMP17]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[TMP18]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[EVL_BASED_IV]], 3
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP19]]
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i8 0
; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP6]], 3
-; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP9]])
-; NO-REG-PRESSURE-CHECK-NEXT: call void @llvm.vp.store.nxv48i8.p0(<vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 48 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
-; NO-REG-PRESSURE-CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
-; NO-REG-PRESSURE-CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[TMP7]]
-; NO-REG-PRESSURE-CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 24 x i8> @llvm.vector.interleave3.nxv24i8(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x i8> [[WIDE_MASKED_GATHER9]], <vscale x 8 x i8> [[WIDE_MASKED_GATHER10]])
+; NO-REG-PRESSURE-CHECK-NEXT: call void @llvm.vp.store.nxv24i8.p0(<vscale x 24 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP21]], <vscale x 24 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; NO-REG-PRESSURE-CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[EVL_BASED_IV]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT9]] = add <vscale x 8 x i64> [[VEC_IND3]], [[BROADCAST_SPLAT4]]
+; NO-REG-PRESSURE-CHECK-NEXT: [[VEC_IND_NEXT10]] = add <vscale x 8 x i64> [[VEC_IND4]], [[BROADCAST_SPLAT]]
; NO-REG-PRESSURE-CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-REG-PRESSURE-CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; NO-REG-PRESSURE-CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 441e9b69bf663..9d5294d14e7cd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -342,30 +342,33 @@ exit:
define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; IF-EVL-LABEL: @load_factor_4_with_tail_gap(
; IF-EVL-NEXT: entry:
-; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4
-; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 8
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
+; IF-EVL-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[INDEX]], 4
-; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP1]]
+; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[INDEX]], 4
+; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 [[TMP3]]
; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP1]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
-; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP1]], i64 [[TMP1]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
-; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
+; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; IF-EVL: middle.block:
@@ -557,39 +560,37 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
; IF-EVL-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
-; IF-EVL-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 4
-; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP2]]
-; IF-EVL-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
-; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
-; IF-EVL-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 8
-; IF-EVL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
-; IF-EVL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP2]], 12
-; IF-EVL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
+; IF-EVL-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP3:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP2]]
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL: vector.body:
-; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP9:%.*]] = mul i64 [[INDEX]], -16
-; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP9]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; IF-EVL-NEXT: [[TMP8:%.*]] = sub nsw i64 0, [[TMP7]]
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP9]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]
-; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP9]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
-; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP9]]
-; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; IF-EVL-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]
; IF-EVL-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])
-; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; IF-EVL: middle.block:
@@ -603,14 +604,6 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
; NO-VP-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
-; NO-VP-NEXT: [[TMP7:%.*]] = shl i64 [[N]], 4
-; NO-VP-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP7]]
-; NO-VP-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
-; NO-VP-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[TMP7]], 8
-; NO-VP-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP15]]
-; NO-VP-NEXT: [[TMP23:%.*]] = add nuw nsw i64 [[TMP7]], 12
-; NO-VP-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP23]]
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
@@ -621,25 +614,32 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]]
-; NO-VP-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
+; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT: [[TMP8:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
+; NO-VP-NEXT: [[TMP9:%.*]] = sub nsw i64 0, [[TMP5]]
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
; NO-VP: vector.body:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP8]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT: [[TMP26:%.*]] = mul i64 [[INDEX]], -16
-; NO-VP-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP26]]
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; NO-VP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP26]]
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 1
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]
-; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP26]]
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; NO-VP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]
-; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP26]]
-; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; NO-VP-NEXT: [[TMP23:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP23]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; NO-VP-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; NO-VP: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index a984f776c31a5..b2a19541d636a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -284,35 +284,35 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK-NEXT: [[CONFLICT_RDX12:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT11]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i16> @llvm.stepvector.nxv2i16()
-; CHECK-NEXT: [[TMP10:%.*]] = mul <vscale x 2 x i16> [[TMP9]], splat (i16 4)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i16> @llvm.stepvector.nxv2i16()
+; CHECK-NEXT: [[TMP8:%.*]] = mul <vscale x 2 x i16> [[TMP7]], splat (i16 4)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i16> [ [[TMP10]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i16> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 256, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i16
-; CHECK-NEXT: [[TMP13:%.*]] = shl i16 [[TMP12]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP10:%.*]] = trunc i32 [[TMP9]] to i16
+; CHECK-NEXT: [[TMP11:%.*]] = shl i16 [[TMP10]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[TMP11]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = sext <vscale x 2 x i16> [[VEC_IND]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP17:%.*]] = shl i32 [[EVL_BASED_IV]], 4
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P1]], i32 [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i32(ptr align 4 [[TMP18]], i32 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
-; CHECK-NEXT: [[TMP20:%.*]] = trunc <vscale x 2 x i32> [[TMP19]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [1 x [1 x i16]], ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i16.nxv2p0(<vscale x 2 x i16> [[TMP20]], <vscale x 2 x ptr> align 2 [[TMP21]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
-; CHECK-NEXT: [[TMP22:%.*]] = trunc <vscale x 2 x i32> [[TMP19]] to <vscale x 2 x i8>
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP22]], <vscale x 2 x ptr> align 1 [[TMP23]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [1 x i64], ptr [[P2]], <vscale x 2 x i64> [[TMP16]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x ptr> align 8 [[TMP24]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META13]]
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP11]], [[EVL_BASED_IV]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = sext <vscale x 2 x i16> [[VEC_IND]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP13:%.*]] = shl i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P1]], i32 [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i32(ptr align 4 [[TMP14]], i32 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT: [[TMP16:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [1 x [1 x i16]], ptr [[P2]], <vscale x 2 x i64> [[TMP12]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i16.nxv2p0(<vscale x 2 x i16> [[TMP16]], <vscale x 2 x ptr> align 2 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT: [[TMP18:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P2]], <vscale x 2 x i64> [[TMP12]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP18]], <vscale x 2 x ptr> align 1 [[TMP19]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [1 x i64], ptr [[P2]], <vscale x 2 x i64> [[TMP12]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x ptr> align 8 [[TMP20]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META13]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP9]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i16> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
>From ccd7158cf8380baf723f7ada6277cd6c0af893bf Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 11 Mar 2026 02:31:36 -0700
Subject: [PATCH 30/35] update comment. nfc
---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 75e629d5dfdac..3da4bf7d57984 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2283,7 +2283,7 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
/// A recipe to compute the pointers for widened memory accesses of \p
/// SourceElementTy, with the \p Stride expressed in units of \p
/// SourceElementTy. Unrolling adds an extra \p VFxPart operand for unrolled
-/// parts > 0 and it produces `GEP Ptr, VFxPart * Stride`.
+/// parts > 0 and it produces `GEP SourceElementTy Ptr, VFxPart * Stride`.
class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 42b9fb578ade7..1402c5cbd48a4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2784,6 +2784,7 @@ void VPVectorPointerRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
Value *Ptr = State.get(getOperand(0), VPLane(0));
Value *Offset = State.get(getVFxPart(), true);
+ // TODO: Expand to VPInstruction to support constant folding.
if (!match(getStride(), m_One())) {
Value *Stride = Builder.CreateZExtOrTrunc(State.get(getStride(), true),
Offset->getType());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d9209a3aa2360..13c3ffb4e88cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6579,7 +6579,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
const SCEV *Start;
const APInt *Step;
- // TODO: Support loop invariant stride.
+ // TODO: Support non-constant loop invariant stride.
if (!match(PtrSCEV,
m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step))))
continue;
@@ -6588,7 +6588,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
if (!isa<SCEVUnknown, SCEVConstant>(Start))
continue;
- // Add VF of i32 version for EVL.
+ // Get VF as i32 for the vector length operand.
if (!I32VF) {
VPBuilder Builder(Plan.getVectorPreheader());
I32VF = Builder.createScalarZExtOrTrunc(
>From 3e412f612e608d32df0a24352829f3f9dae8bf81 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 12 Mar 2026 00:08:50 -0700
Subject: [PATCH 31/35] Pass Intrrinsic ID into constructor
---
.../Transforms/Vectorize/LoopVectorizationPlanner.h | 7 ++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 12 ++++++++----
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 +++++----
3 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 3f583f1098840..d3adda1430788 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -442,10 +442,11 @@ class VPBuilder {
}
VPWidenMemIntrinsicRecipe *
- createWidenMemIntrinsic(LoadInst &LI, ArrayRef<VPValue *> CallArguments,
+ createWidenMemIntrinsic(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments,
const VPIRMetadata &MD, DebugLoc DL) {
- return tryInsertInstruction(
- new VPWidenMemIntrinsicRecipe(LI, CallArguments, MD, DL));
+ return tryInsertInstruction(new VPWidenMemIntrinsicRecipe(
+ LI, VectorIntrinsicID, CallArguments, MD, DL));
}
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3da4bf7d57984..73da77ebf2a0c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2019,13 +2019,16 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
public:
// TODO: support StoreInst for strided store
- VPWidenMemIntrinsicRecipe(LoadInst &LI, ArrayRef<VPValue *> CallArguments,
+ VPWidenMemIntrinsicRecipe(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments,
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPWidenIntrinsicRecipe(VPRecipeBase::VPWidenMemIntrinsicSC,
- Intrinsic::experimental_vp_strided_load,
- CallArguments, LI.getType(), {}, MD, DL),
+ VectorIntrinsicID, CallArguments, LI.getType(),
+ {}, MD, DL),
Alignment(LI.getAlign()) {
+ assert(VectorIntrinsicID == Intrinsic::experimental_vp_strided_load &&
+ "Unexpected intrinsic");
setUnderlyingValue(&LI);
}
@@ -2033,7 +2036,8 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
VPWidenMemIntrinsicRecipe *clone() override {
return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(getUnderlyingInstr()),
- operands(), *this, getDebugLoc());
+ getVectorIntrinsicID(), operands(),
+ *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemIntrinsicSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 13c3ffb4e88cd..9eeac9004dc0e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2944,8 +2944,9 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
if (!Mask)
Mask = Plan->getTrue();
return new VPWidenMemIntrinsicRecipe(
- *cast<LoadInst>(I->getUnderlyingInstr()), {Addr, Stride, Mask, &EVL},
- *I, I->getDebugLoc());
+ *cast<LoadInst>(I->getUnderlyingInstr()),
+ Intrinsic::experimental_vp_strided_load, {Addr, Stride, Mask, &EVL}, *I,
+ I->getDebugLoc());
}
VPValue *StoredVal;
@@ -6620,8 +6621,8 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
if (!Mask)
Mask = Plan.getTrue();
auto *StridedLoad = Builder.createWidenMemIntrinsic(
- *cast<LoadInst>(&Ingredient), {NewPtr, StrideInBytes, Mask, I32VF},
- *LoadR, LoadR->getDebugLoc());
+ *cast<LoadInst>(&Ingredient), Intrinsic::experimental_vp_strided_load,
+ {NewPtr, StrideInBytes, Mask, I32VF}, *LoadR, LoadR->getDebugLoc());
LoadR->replaceAllUsesWith(StridedLoad);
ToErase.push_back(LoadR);
>From 01c3ca40437ad1ff0a18b2ad9dc025f63891a679 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 12 Mar 2026 02:01:17 -0700
Subject: [PATCH 32/35] Remove underlying
---
.../Vectorize/LoopVectorizationPlanner.h | 9 ++++----
llvm/lib/Transforms/Vectorize/VPlan.h | 19 ++++++++-------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 23 +++++++++----------
4 files changed, 27 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d3adda1430788..eb08ff1f8d754 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -441,12 +441,11 @@ class VPBuilder {
new VPVectorPointerRecipe(Ptr, SourceElementTy, Stride, GEPFlags, DL));
}
- VPWidenMemIntrinsicRecipe *
- createWidenMemIntrinsic(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
- ArrayRef<VPValue *> CallArguments,
- const VPIRMetadata &MD, DebugLoc DL) {
+ VPWidenMemIntrinsicRecipe *createWidenMemIntrinsic(
+ Intrinsic::ID VectorIntrinsicID, ArrayRef<VPValue *> CallArguments,
+ Type *Ty, Align Alignment, const VPIRMetadata &MD, DebugLoc DL) {
return tryInsertInstruction(new VPWidenMemIntrinsicRecipe(
- LI, VectorIntrinsicID, CallArguments, MD, DL));
+ VectorIntrinsicID, CallArguments, Ty, Alignment, MD, DL));
}
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 73da77ebf2a0c..4476bc185767e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2019,25 +2019,24 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
public:
// TODO: support StoreInst for strided store
- VPWidenMemIntrinsicRecipe(LoadInst &LI, Intrinsic::ID VectorIntrinsicID,
- ArrayRef<VPValue *> CallArguments,
- const VPIRMetadata &MD = {},
+ VPWidenMemIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments, Type *Ty,
+ Align Alignment, const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPWidenIntrinsicRecipe(VPRecipeBase::VPWidenMemIntrinsicSC,
- VectorIntrinsicID, CallArguments, LI.getType(),
- {}, MD, DL),
- Alignment(LI.getAlign()) {
+ VectorIntrinsicID, CallArguments, Ty, {}, MD,
+ DL),
+ Alignment(Alignment) {
assert(VectorIntrinsicID == Intrinsic::experimental_vp_strided_load &&
"Unexpected intrinsic");
- setUnderlyingValue(&LI);
}
~VPWidenMemIntrinsicRecipe() override = default;
VPWidenMemIntrinsicRecipe *clone() override {
- return new VPWidenMemIntrinsicRecipe(*cast<LoadInst>(getUnderlyingInstr()),
- getVectorIntrinsicID(), operands(),
- *this, getDebugLoc());
+ return new VPWidenMemIntrinsicRecipe(getVectorIntrinsicID(), operands(),
+ getResultType(), Alignment, *this,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemIntrinsicSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1402c5cbd48a4..e4d7335cdd1da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1929,7 +1929,7 @@ CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
assert(VectorF &&
"Can't retrieve vector intrinsic or vector-predication intrinsics.");
- auto *CI = dyn_cast_or_null<CallInst>(getUnderlyingValue());
+ auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
SmallVector<OperandBundleDef, 1> OpBundles;
if (CI)
CI->getOperandBundlesAsDefs(OpBundles);
@@ -2048,12 +2048,9 @@ InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
Type *Ty = toVectorTy(getResultType(), VF);
- const Instruction *Ingredient = getUnderlyingInstr();
- const Value *Ptr = getLoadStorePointerOperand(Ingredient);
return Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, Ptr,
- match(getOperand(2), m_True()), Alignment,
- Ingredient),
+ MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, /*Ptr=*/nullptr,
+ !match(getOperand(2), m_True()), Alignment),
Ctx.CostKind);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9eeac9004dc0e..9a9db922a0a20 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2940,13 +2940,12 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
m_VPValue(Addr), m_VPValue(Stride),
m_RemoveMask(HeaderMask, Mask),
m_TruncOrSelf(m_Specific(&Plan->getVF()))))) {
- auto *I = cast<VPWidenIntrinsicRecipe>(&CurRecipe);
if (!Mask)
Mask = Plan->getTrue();
- return new VPWidenMemIntrinsicRecipe(
- *cast<LoadInst>(I->getUnderlyingInstr()),
- Intrinsic::experimental_vp_strided_load, {Addr, Stride, Mask, &EVL}, *I,
- I->getDebugLoc());
+ auto *NewLoad = cast<VPWidenMemIntrinsicRecipe>(&CurRecipe)->clone();
+ NewLoad->setOperand(2, Mask);
+ NewLoad->setOperand(3, &EVL);
+ return NewLoad;
}
VPValue *StoredVal;
@@ -6556,10 +6555,10 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
if (!Ptr)
continue;
- Instruction &Ingredient = LoadR->getIngredient();
+ Type *LoadTy = TypeInfo.inferScalarType(LoadR);
+ Align Alignment = LoadR->getAlign();
auto IsProfitable = [&](ElementCount VF) -> bool {
- Type *DataTy = toVectorTy(getLoadStoreType(&Ingredient), VF);
- const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ Type *DataTy = toVectorTy(LoadTy, VF);
if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment))
return false;
const InstructionCost CurrentCost = LoadR->computeCost(VF, Ctx);
@@ -6567,8 +6566,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::experimental_vp_strided_load, DataTy,
- Ptr->getUnderlyingValue(), LoadR->isMasked(), Alignment,
- &Ingredient),
+ /*Ptr=*/nullptr, LoadR->isMasked(), Alignment),
Ctx.CostKind);
return StridedLoadStoreCost < CurrentCost;
};
@@ -6621,8 +6619,9 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
if (!Mask)
Mask = Plan.getTrue();
auto *StridedLoad = Builder.createWidenMemIntrinsic(
- *cast<LoadInst>(&Ingredient), Intrinsic::experimental_vp_strided_load,
- {NewPtr, StrideInBytes, Mask, I32VF}, *LoadR, LoadR->getDebugLoc());
+ Intrinsic::experimental_vp_strided_load,
+ {NewPtr, StrideInBytes, Mask, I32VF}, LoadTy, Alignment, *LoadR,
+ LoadR->getDebugLoc());
LoadR->replaceAllUsesWith(StridedLoad);
ToErase.push_back(LoadR);
>From 124888c97847fc032153b77c0a0987862ee34e44 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 20 Apr 2026 06:18:58 -0700
Subject: [PATCH 33/35] remove type annotation
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9a9db922a0a20..477728f2d3089 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6557,7 +6557,7 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Type *LoadTy = TypeInfo.inferScalarType(LoadR);
Align Alignment = LoadR->getAlign();
- auto IsProfitable = [&](ElementCount VF) -> bool {
+ auto IsProfitable = [&](ElementCount VF) {
Type *DataTy = toVectorTy(LoadTy, VF);
if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment))
return false;
>From 616d922c33b6315fc19814e46bd0d1df44ff2a5c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 5 May 2026 03:18:27 -0700
Subject: [PATCH 34/35] Check strided access pattern before profitable check.
This can reduce the ineffective divergence plan
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++++++++---------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 477728f2d3089..998db5d67e876 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6555,6 +6555,20 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
if (!Ptr)
continue;
+ // Check if this is a strided access by analyzing the address SCEV for an
+ // affine add recurrence pattern.
+ const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
+ const SCEV *Start;
+ const APInt *Step;
+ // TODO: Support non-constant loop invariant stride.
+ if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step),
+ m_SpecificLoop(&L))))
+ continue;
+
+ // TODO: Support the base pointer that requires SCEV expander.
+ if (!isa<SCEVUnknown, SCEVConstant>(Start))
+ continue;
+
Type *LoadTy = TypeInfo.inferScalarType(LoadR);
Align Alignment = LoadR->getAlign();
auto IsProfitable = [&](ElementCount VF) {
@@ -6575,18 +6589,6 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Range))
continue;
- const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, &L);
- const SCEV *Start;
- const APInt *Step;
- // TODO: Support non-constant loop invariant stride.
- if (!match(PtrSCEV,
- m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step))))
- continue;
-
- // TODO: Support the base pointer that requires SCEV expander.
- if (!isa<SCEVUnknown, SCEVConstant>(Start))
- continue;
-
// Get VF as i32 for the vector length operand.
if (!I32VF) {
VPBuilder Builder(Plan.getVectorPreheader());
>From 7100e99e9b2c29fefd2adcb4abe4c405eabf46e5 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 5 May 2026 06:48:16 -0700
Subject: [PATCH 35/35] compute cost by static function
---
llvm/lib/Transforms/Vectorize/VPlan.h | 5 +++++
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +++++++++++----
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++-----
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4476bc185767e..496c8ccdf6339 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2044,6 +2044,11 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
/// Produce a widened version of the vector memory intrinsic.
void execute(VPTransformState &State) override;
+ /// Helper function for computing the cost of vector memory intrinsic.
+ static InstructionCost computeMemIntrinsicCost(Intrinsic::ID IID, Type *Ty,
+ bool IsMasked, Align Alignment,
+ VPCostContext &Ctx);
+
/// Return the cost of this vector memory intrinsic.
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e4d7335cdd1da..d18b5d7ee0543 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2044,14 +2044,21 @@ void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
State.set(this, MemI);
}
+InstructionCost VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
+ Intrinsic::ID IID, Type *Ty, bool IsMasked, Align Alignment,
+ VPCostContext &Ctx) {
+ return Ctx.TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(IID, Ty, /*Ptr=*/nullptr, IsMasked, Alignment),
+ Ctx.CostKind);
+}
+
InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
Type *Ty = toVectorTy(getResultType(), VF);
- return Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(getVectorIntrinsicID(), Ty, /*Ptr=*/nullptr,
- !match(getOperand(2), m_True()), Alignment),
- Ctx.CostKind);
+ return computeMemIntrinsicCost(getVectorIntrinsicID(), Ty,
+ !match(getOperand(2), m_True()), Alignment,
+ Ctx);
}
void VPHistogramRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 998db5d67e876..4646b50f04240 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6577,11 +6577,9 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
return false;
const InstructionCost CurrentCost = LoadR->computeCost(VF, Ctx);
const InstructionCost StridedLoadStoreCost =
- Ctx.TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(
- Intrinsic::experimental_vp_strided_load, DataTy,
- /*Ptr=*/nullptr, LoadR->isMasked(), Alignment),
- Ctx.CostKind);
+ VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
+ Intrinsic::experimental_vp_strided_load, DataTy,
+ LoadR->isMasked(), Alignment, Ctx);
return StridedLoadStoreCost < CurrentCost;
};
More information about the llvm-commits
mailing list