[llvm] [VPlan] Assert that replacement types match in VPUser::setOperand (NFC). (PR #195891)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 10:51:54 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/195891
Add assertion to VPValue::setOperand to check if types of the new
operand matches the old operand.
This makes it easier to catch replacements with incorrect types at the
source, instead only later during verification.
A few places currently preform replacements with mis-matching types,
which only get fixed up later. Update those to avoid type-violation.
Depends on https://github.com/llvm/llvm-project/pull/195485 (included in
PR)
>From da31eadf93de02c761d2efaa0db335d5447075a7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 19 Apr 2026 10:34:39 +0100
Subject: [PATCH 1/4] [VPlan] Add Type* and getScalarType() to VPRecipeValue
and VPSymbolicValue
Add a Type* field to VPRecipeValue and VPSymbolicValue, along with
getScalarType() methods to query the stored scalar type. Add a
VPValue::getScalarType() dispatch method that handles all VPValue
subclasses (VPIRValue, VPSymbolicValue, VPRecipeValue).
Set scalar types on VPSymbolicValues (VF, VFxUF, VectorTripCount, UF)
when the canonical IV is created in addCanonicalIVRecipes, and propagate
the type to BackedgeTakenCount when it is created. Propagate types
during VPlan::duplicate.
This is the first step towards storing types directly on VPValues
instead of relying solely on VPTypeAnalysis::inferScalarType.
fixup! [VPlan] Add Type* and getScalarType() to VPRecipeValue and VPSymbolicValue
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 9 ++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 14 ++++++----
.../Transforms/Vectorize/VPlanAnalysis.cpp | 26 +++----------------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 8 ++----
.../Vectorize/VPlanConstruction.cpp | 7 ++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 9 ++++++-
7 files changed, 33 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 453b5eec9fe5e..556928696cd23 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -862,7 +862,8 @@ VPInstruction *VPRegionBlock::getOrCreateCanonicalIVIncrement() {
CanIV->getDebugLoc(), "index.next");
}
-VPlan::VPlan(Loop *L) {
+VPlan::VPlan(Loop *L, Type *IdxTy)
+ : VectorTripCount(IdxTy), VF(IdxTy), UF(IdxTy), VFxUF(IdxTy) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
@@ -1228,7 +1229,8 @@ VPlan *VPlan::duplicate() {
NewScalarHeader = createVPIRBasicBlock(ScalarHeaderIRBB);
}
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
- auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+ auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader,
+ VF.getType());
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPIRValue *OldLiveIn : getLiveIns())
Old2NewVPValues[OldLiveIn] = NewPlan->getOrAddLiveIn(OldLiveIn);
@@ -1250,7 +1252,8 @@ VPlan *VPlan::duplicate() {
"All VPSymbolicValues must be handled below");
if (BackedgeTakenCount)
- NewPlan->BackedgeTakenCount = new VPSymbolicValue();
+ NewPlan->BackedgeTakenCount =
+ new VPSymbolicValue(BackedgeTakenCount->getType());
// Map and propagate materialized state for symbolic values.
for (auto [OldSV, NewSV] :
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..01c8585025dd7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4567,8 +4567,10 @@ class VPlan {
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
/// wrapping the original header of the scalar loop.
- VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
- : Entry(Entry), ScalarHeader(ScalarHeader) {
+ VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader,
+ Type *IdxTy = nullptr)
+ : Entry(Entry), ScalarHeader(ScalarHeader), VectorTripCount(IdxTy),
+ VF(IdxTy), UF(IdxTy), VFxUF(IdxTy) {
Entry->setPlan(this);
assert(ScalarHeader->getNumSuccessors() == 0 &&
"scalar header must be a leaf node");
@@ -4578,11 +4580,12 @@ class VPlan {
/// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
/// original preheader and scalar header of \p L, to be used as entry and
/// scalar header blocks of the new VPlan.
- VPlan(Loop *L);
+ VPlan(Loop *L, Type *IdxTy = nullptr);
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
/// wrapping \p ScalarHeaderBB and a trip count of \p TC.
- VPlan(BasicBlock *ScalarHeaderBB) {
+ VPlan(BasicBlock *ScalarHeaderBB, Type *IdxTy = nullptr)
+ : VectorTripCount(IdxTy), VF(IdxTy), UF(IdxTy), VFxUF(IdxTy) {
setEntry(createVPBasicBlock("preheader"));
ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
}
@@ -4681,8 +4684,9 @@ class VPlan {
/// The backedge taken count of the original loop.
VPValue *getOrCreateBackedgeTakenCount() {
+ // BTC shares the canonical IV type with VectorTripCount.
if (!BackedgeTakenCount)
- BackedgeTakenCount = new VPSymbolicValue();
+ BackedgeTakenCount = new VPSymbolicValue(VectorTripCount.getType());
return BackedgeTakenCount;
}
VPValue *getBackedgeTakenCount() const { return BackedgeTakenCount; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 103dff1889a6a..7bb93a2164279 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -24,23 +24,6 @@ using namespace VPlanPatternMatch;
#define DEBUG_TYPE "vplan"
-VPTypeAnalysis::VPTypeAnalysis(const VPlan &Plan)
- : Ctx(Plan.getContext()), DL(Plan.getDataLayout()) {
- if (auto LoopRegion = Plan.getVectorLoopRegion()) {
- CanonicalIVTy = LoopRegion->getCanonicalIVType();
- return;
- }
-
- // If there's no loop region, retrieve the type from the trip count
- // expression.
- auto *TC = Plan.getTripCount();
- if (auto *TCIRV = dyn_cast<VPIRValue>(TC)) {
- CanonicalIVTy = TCIRV->getType();
- return;
- }
- CanonicalIVTy = cast<VPExpandSCEVRecipe>(TC)->getSCEV()->getType();
-}
-
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPBlendRecipe *R) {
Type *ResTy = inferScalarType(R->getIncomingValue(0));
for (unsigned I = 1, E = R->getNumIncomingValues(); I != E; ++I) {
@@ -121,7 +104,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::LastActiveLane:
// Assume that the maximum possible number of elements in a vector fits
// within the index type for the default address space.
- return DL.getIndexType(Ctx, 0);
+ return R->getParent()->getPlan()->getDataLayout().getIndexType(Ctx, 0);
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
assert(inferScalarType(R->getOperand(0))->isIntegerTy(1) &&
@@ -279,11 +262,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (auto *IRV = dyn_cast<VPIRValue>(V))
return IRV->getType();
- if (isa<VPSymbolicValue>(V)) {
- // All VPValues without any underlying IR value (like the vector trip count
- // or the backedge-taken count) have the same type as the canonical IV.
- return CanonicalIVTy;
- }
+ if (auto *SymbolicV = dyn_cast<VPSymbolicValue>(V))
+ return SymbolicV->getType();
if (auto *RegionV = dyn_cast<VPRegionValue>(V))
return RegionV->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index c1c9075420d1c..b73636c31f0a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -9,6 +9,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
#define LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
+#include "VPlan.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
@@ -45,12 +46,7 @@ struct VPCostContext;
/// of the previously inferred types.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
- /// Type of the canonical induction variable. Used for all VPValues without
- /// any underlying IR value (like the vector trip count or the backedge-taken
- /// count).
- Type *CanonicalIVTy;
LLVMContext &Ctx;
- const DataLayout &DL;
Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
Type *inferScalarTypeForRecipe(const VPInstruction *R);
@@ -60,7 +56,7 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
- VPTypeAnalysis(const VPlan &Plan);
+ VPTypeAnalysis(const VPlan &Plan) : Ctx(Plan.getContext()) {}
/// Infer the type of \p V. Returns the scalar type of \p V.
Type *inferScalarType(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e20d5d947ac54..c2b05984db272 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -78,8 +78,9 @@ class PlainCFGBuilder {
void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
public:
- PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer)
- : TheLoop(Lp), LI(LI), LVer(LVer), Plan(std::make_unique<VPlan>(Lp)) {}
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer, Type *IdxTy)
+ : TheLoop(Lp), LI(LI), LVer(LVer),
+ Plan(std::make_unique<VPlan>(Lp, IdxTy)) {}
/// Build plain CFG for TheLoop and connect it to Plan's entry.
std::unique_ptr<VPlan> buildPlainCFG();
@@ -635,7 +636,7 @@ std::unique_ptr<VPlan>
VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
DebugLoc IVDL, PredicatedScalarEvolution &PSE,
LoopVersioning *LVer) {
- PlainCFGBuilder Builder(TheLoop, &LI, LVer);
+ PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy);
std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop);
simplifyLiveInsWithSCEV(*VPlan0, PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f1b9efae08377..30dd61ba5b232 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -345,7 +345,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
VPBuilder Builder(VPR);
const DataLayout &DL = Plan.getDataLayout();
Type *IndexTy = DL.getIndexType(TypeInfo.inferScalarType(VPR));
- Type *VFTy = TypeInfo.inferScalarType(&Plan.getVF());
+ Type *VFTy = Plan.getVF().getType();
VPValue *VF = Builder.createScalarZExtOrTrunc(
&Plan.getVF(), IndexTy, VFTy, DebugLoc::getUnknown());
// VFxUF does not wrap, so VF * Part also cannot wrap.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8f9806adf774e..1e79c48cc8f7c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -278,12 +278,16 @@ struct VPConstantInt : public VPIRValue {
/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
/// VFxUF.
struct VPSymbolicValue : public VPValue {
- VPSymbolicValue() : VPValue(VPVSymbolicSC, nullptr) {}
+ VPSymbolicValue(Type *Ty = nullptr)
+ : VPValue(VPVSymbolicSC, nullptr), Ty(Ty) {}
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVSymbolicSC;
}
+ /// Returns the scalar type of this symbolic value.
+ Type *getType() const { return Ty; }
+
/// Returns true if this symbolic value has been materialized.
bool isMaterialized() const { return Materialized; }
@@ -294,6 +298,9 @@ struct VPSymbolicValue : public VPValue {
}
private:
+ /// The scalar type of this symbolic value.
+ Type *Ty;
+
/// Track whether this symbolic value has been materialized (replaced).
/// After materialization, accessing users should trigger an assertion.
bool Materialized = false;
>From 1481802a2b1c51f88f5f55d253a09f121a678908 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 11 Apr 2026 17:19:53 +0100
Subject: [PATCH 2/4] [VPlan] Remove Def pointer from VPRecipeValue for
single-def recipes
For VPSingleDefRecipe, the VPRecipeValue's Def pointer always points
back to the containing VPRecipeBase, which is computable via
static_cast.
Introduce 2 VPRecipeValue subclasses to distinguish the VPValues defined
by VPSingleDefRecipes (VPSingleDefValue), and VPStandaloneValue for
other recipes.
The former does not need to store a pointer to the defining recipe, as
it can be computed via static_cast. This saves 8 bytes for most recipes.
I plan to use the extra bytes to store the type directly in VPValue as
follow-ups.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 31 +++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 22 +++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 70 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
4 files changed, 89 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 556928696cd23..23f7feb8f41a8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -122,17 +122,22 @@ void VPRecipeBase::dump() const {
#endif
#if !defined(NDEBUG)
-bool VPRecipeValue::isDefinedBy(const VPDef *D) const { return Def == D; }
+bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
+ return getDefiningRecipe() == D;
+}
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ if (!DefValue)
+ return nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
+ return SV->getDef();
+ return static_cast<VPSingleDefRecipe *>(DefValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ return const_cast<VPValue *>(this)->getDefiningRecipe();
}
Value *VPValue::getLiveInIRValue() const {
@@ -141,15 +146,21 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::VPRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPValue(VPVRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPRecipeValue requires a defining recipe");
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
+ : VPRecipeValue(VPVSingleDefValueSC, UV) {
+ assert(Def && "VPSingleDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPRecipeValue::~VPRecipeValue() = default;
+
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+ assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() {
- assert(Users.empty() &&
- "trying to delete a VPRecipeValue with remaining users");
+VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
Def->removeDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 01c8585025dd7..4c47c6ed973e8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -605,16 +605,17 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
-/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+
+ ~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -2873,7 +2874,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst);
}
} else {
for (auto *SV : StoredValues)
@@ -3575,13 +3576,14 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
+ : public VPWidenMemoryRecipe,
+ public VPStandaloneRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3616,13 +3618,13 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// using the address to load from, the explicit vector length and an optional
/// mask.
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+ public VPStandaloneRecipeValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 1e79c48cc8f7c..c7e2b299fe48a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -39,6 +39,7 @@ class VPDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
+class VPSingleDefRecipe;
class VPPhiAccessors;
class VPRegionValue;
class VPRegionBlock;
@@ -79,11 +80,12 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
- VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVRecipeValueSC, /// A VPValue defined by a recipe.
- VPRegionValueSC, /// A VPValue sub-class that is defined by a region, like
- /// the canonical IV of a loop region.
+ VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVStandaloneRecipeValueSC, /// A standalone VPValue defined by a recipe.
+ VPVSingleDefValueSC, /// A VPValue embedded in a VPSingleDefRecipe.
+ VPRegionValueSC, /// A VPValue sub-class that is defined by a
+ /// region, like a loop region canonical IV.
};
VPValue(const VPValue &) = delete;
@@ -306,14 +308,11 @@ struct VPSymbolicValue : public VPValue {
bool Materialized = false;
};
-/// A VPValue defined by a recipe that produces one or more values.
+/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
- /// Pointer to the VPRecipeBase that defines this VPValue.
- VPRecipeBase *Def;
-
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -321,13 +320,51 @@ class VPRecipeValue : public VPValue {
bool isDefinedBy(const VPDef *D) const;
#endif
+protected:
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+
+public:
+ LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
+ V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue embedded as a subobject of VPSingleDefRecipe.
+class VPSingleDefValue : public VPRecipeValue {
+ friend class VPDef;
+ friend class VPSingleDefRecipe;
+
+protected:
+ /// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
+ LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
+ Value *UV = nullptr);
+
+public:
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue that stores a pointer to its defining recipe.
+class VPStandaloneRecipeValue : public VPRecipeValue {
+ friend class VPDef;
+
+ /// Pointer to the VPRecipeBase that defines this VPValue.
+ VPRecipeBase *Def;
+
public:
- LLVM_ABI_FOR_TEST VPRecipeValue(VPRecipeBase *Def, Value *UV = nullptr);
+ LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
+ Value *UV = nullptr);
- LLVM_ABI_FOR_TEST virtual ~VPRecipeValue();
+ ~VPStandaloneRecipeValue() override;
+
+ VPRecipeBase *getDef() const { return Def; }
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVRecipeValueSC;
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC;
}
};
@@ -437,15 +474,15 @@ class VPUser {
/// Single-value VPDefs that also inherit from VPValue must make sure to inherit
/// from VPDef before VPValue.
class VPDef {
- friend class VPRecipeValue;
+ friend class VPSingleDefValue;
+ friend class VPSingleDefRecipe;
+ friend class VPStandaloneRecipeValue;
/// The VPValues defined by this VPDef.
TinyPtrVector<VPRecipeValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
void addDefinedValue(VPRecipeValue *V) {
- assert(V->isDefinedBy(this) &&
- "can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
}
@@ -457,7 +494,8 @@ class VPDef {
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
llvm::erase(DefinedValues, V);
- V->Def = nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(V))
+ SV->Def = nullptr;
}
public:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a1ddda7eda969..9cc1d90d6256d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1698,8 +1698,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPRecipeValue(this);
- new VPRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
>From 76f4cccb67b8ced5a6bbab4ab53022d0244691c6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 1 May 2026 14:18:29 +0100
Subject: [PATCH 3/4] [VPlan] Store scalar type in VPRecipeValue, migrae first
set of recipes.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 24 +++-
llvm/lib/Transforms/Vectorize/VPlan.h | 111 +++++++++++-------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 47 ++------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 4 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 18 ++-
.../Transforms/Vectorize/VPlanVerifier.cpp | 56 +++++++++
.../Transforms/Vectorize/VPlanTest.cpp | 7 +-
.../Vectorize/VPlanVerifierTest.cpp | 38 ++++++
11 files changed, 240 insertions(+), 102 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ee90c09085c3f..9120409ae60d9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -326,7 +326,7 @@ cl::opt<bool>
#ifdef EXPENSIVE_CHECKS
cl::init(true),
#else
- cl::init(false),
+ cl::init(true),
#endif
cl::Hidden,
cl::desc("Verify VPlans after VPlan transforms."));
@@ -7257,7 +7257,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
ToDelete.push_back(AnyOfSelect);
// Convert the reduction phi to operate on bools.
- PhiR->setOperand(0, Plan->getFalse());
+ auto *NewPhiR =
+ PhiR->cloneWithOperands(Plan->getFalse(), PhiR->getBackedgeValue());
+ NewPhiR->insertBefore(PhiR);
+ PhiR->replaceAllUsesWith(NewPhiR);
+ PhiR = NewPhiR;
// Update NewExitingVPV if it was pointing to the now-replaced select.
if (NewExitingVPV == AnyOfSelect)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 23f7feb8f41a8..fb15c3f78d89c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -146,16 +146,32 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
- : VPRecipeValue(VPVSingleDefValueSC, UV) {
+Type *VPValue::getScalarType() const {
+ switch (getVPValueID()) {
+ case VPVIRValueSC:
+ return getUnderlyingValue()->getType();
+ case VPVSymbolicSC:
+ return cast<VPSymbolicValue>(this)->getType();
+ case VPRegionValueSC:
+ return cast<VPRegionValue>(this)->getType();
+ case VPVStandaloneRecipeValueSC:
+ case VPVSingleDefValueSC:
+ return cast<VPRecipeValue>(this)->getScalarType();
+ }
+ llvm_unreachable("Unhandled VPValue subclass");
+}
+
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV, Type *Ty)
+ : VPRecipeValue(VPVSingleDefValueSC, UV, Ty) {
assert(Def && "VPSingleDefValue requires a defining recipe");
Def->addDefinedValue(this);
}
VPRecipeValue::~VPRecipeValue() = default;
-VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV,
+ Type *Ty)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV, Ty), Def(Def) {
assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4c47c6ed973e8..a4c6159b2cdd3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -603,6 +603,11 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return R->getVPRecipeID() == VPRecipeID; \
}
+/// Return the scalar type of \p V. If \p V's scalar type has not been set
+/// (e.g. the defining recipe has not yet been migrated), fall back to
+/// VPTypeAnalysis using the plan of the defining recipe.
+LLVM_ABI Type *getScalarTypeOrInfer(VPValue *V);
+
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
@@ -615,6 +620,11 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+ VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *Ty, Value *UV = nullptr,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV, Ty) {}
+
~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
@@ -1117,6 +1127,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
DebugLoc DL = DebugLoc::getUnknown())
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, const VPIRFlags &Flags,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPSingleDefRecipe(SC, Operands, ResultTy, /*UV=*/nullptr, DL),
+ VPIRFlags(Flags) {}
+
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
@@ -1838,16 +1854,14 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
- /// Result type for the cast.
- Type *ResultTy;
-
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, ResultTy, Flags,
+ DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
assert(hasRequiredFlagsForOpcode(Opcode) &&
@@ -1858,7 +1872,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
+ return new VPWidenCastRecipe(Opcode, getOperand(0), getScalarType(),
cast_or_null<CastInst>(getUnderlyingValue()),
*this, *this, getDebugLoc());
}
@@ -1875,7 +1889,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Instruction::CastOps getOpcode() const { return Opcode; }
/// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1890,9 +1904,6 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
- /// Scalar return type of the intrinsic.
- Type *ResultTy;
-
/// True if the intrinsic may read from memory.
bool MayReadFromMemory;
@@ -1908,9 +1919,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {
@@ -1922,10 +1933,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty) {
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1941,10 +1951,11 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
- operands(), ResultTy, *this, *this,
- getDebugLoc());
- return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
- *this, *this, getDebugLoc());
+ operands(), getResultType(), *this,
+ *this, getDebugLoc());
+ return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(),
+ getResultType(), *this, *this,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntrinsicSC)
@@ -1960,7 +1971,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
/// Return the scalar return type of the intrinsic.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
/// Return to name of the intrinsic as string.
StringRef getIntrinsicName() const;
@@ -1997,8 +2008,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments,
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
- DL),
+ : VPRecipeWithIRFlags(
+ VPRecipeBase::VPWidenCallSC, CallArguments,
+ cast<Function>(CallArguments.back()->getLiveInIRValue())
+ ->getReturnType(),
+ Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
@@ -2099,7 +2113,8 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands,
+ getScalarTypeOrInfer(Operands[0]), Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -2159,7 +2174,7 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
: VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
- GEPFlags, DL),
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy), Stride(Stride) {
assert(Stride < 0 && "Stride must be negative");
}
@@ -2227,7 +2242,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr,
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
@@ -2478,10 +2494,15 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
~VPWidenIntOrFpInductionRecipe() override = default;
+ VPWidenIntOrFpInductionRecipe *cloneWithOperands(VPIRValue *Start,
+ VPValue *Step, VPValue *VF) {
+ return new VPWidenIntOrFpInductionRecipe(getPHINode(), Start, Step, VF,
+ getInductionDescriptor(), Trunc,
+ *this, getDebugLoc());
+ }
+
VPWidenIntOrFpInductionRecipe *clone() override {
- return new VPWidenIntOrFpInductionRecipe(
- getPHINode(), getStartValue(), getStepValue(), getVFValue(),
- getInductionDescriptor(), Trunc, *this, getDebugLoc());
+ return cloneWithOperands(getStartValue(), getStepValue(), getVFValue());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntOrFpInductionSC)
@@ -2586,11 +2607,13 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
std::string Name;
public:
- /// Create a new VPWidenPHIRecipe with incoming values \p IncomingvValues,
+ /// Create a new VPWidenPHIRecipe with incoming values \p IncomingValues,
/// debug location \p DL and \p Name.
VPWidenPHIRecipe(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues, DL),
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues,
+ getScalarTypeOrInfer(IncomingValues[0]),
+ /*UV=*/nullptr, DL),
Name(Name.str()) {}
VPWidenPHIRecipe *clone() override {
@@ -2709,11 +2732,15 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
~VPReductionPHIRecipe() override = default;
- VPReductionPHIRecipe *clone() override {
+ VPReductionPHIRecipe *cloneWithOperands(VPValue *Start,
+ VPValue *BackedgeValue) {
return new VPReductionPHIRecipe(
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
- *getOperand(0), *getBackedgeValue(), Style, *this,
- HasUsesOutsideReductionChain);
+ *Start, *BackedgeValue, Style, *this, HasUsesOutsideReductionChain);
+ }
+
+ VPReductionPHIRecipe *clone() override {
+ return cloneWithOperands(getOperand(0), getBackedgeValue());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPReductionPHISC)
@@ -2874,7 +2901,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPStandaloneRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst, Inst->getType());
}
} else {
for (auto *SV : StoredValues)
@@ -3460,7 +3487,8 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV,
+ getScalarTypeOrInfer(PredV), nullptr, DL) {}
~VPPredInstPHIRecipe() override = default;
VPPredInstPHIRecipe *clone() override {
@@ -3583,7 +3611,7 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPStandaloneRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load, Load.getType()) {
setMask(Mask);
}
@@ -3624,7 +3652,8 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPStandaloneRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient(),
+ getIngredient().getType()) {
setMask(Mask);
}
@@ -3753,8 +3782,7 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
const SCEV *Expr;
public:
- VPExpandSCEVRecipe(const SCEV *Expr)
- : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}), Expr(Expr) {}
+ VPExpandSCEVRecipe(const SCEV *Expr);
~VPExpandSCEVRecipe() override = default;
@@ -3863,7 +3891,8 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
public VPUnrollPartAccessor<1> {
public:
VPWidenCanonicalIVRecipe(VPRegionValue *CanonicalIV)
- : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV}) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV},
+ CanonicalIV->getType(), /*UV=*/nullptr) {}
~VPWidenCanonicalIVRecipe() override = default;
@@ -3980,7 +4009,7 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
Instruction::BinaryOps Opcode, FastMathFlags FMFs,
DebugLoc DL)
: VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC, {IV, Step, VF},
- FMFs, DL),
+ getScalarTypeOrInfer(IV), FMFs, DL),
InductionOpcode(Opcode) {}
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 7bb93a2164279..c742250493cb0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -188,17 +188,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
llvm_unreachable("Unhandled opcode!");
}
-Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) {
- auto &CI = *cast<CallInst>(R->getUnderlyingInstr());
- return CI.getType();
-}
-
-Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) {
- assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)) &&
- "Store recipes should not define any values");
- return cast<LoadInst>(&R->getIngredient())->getType();
-}
-
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
unsigned Opcode = R->getUnderlyingInstr()->getOpcode();
@@ -259,14 +248,15 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (auto *IRV = dyn_cast<VPIRValue>(V))
- return IRV->getType();
-
- if (auto *SymbolicV = dyn_cast<VPSymbolicValue>(V))
- return SymbolicV->getType();
-
- if (auto *RegionV = dyn_cast<VPRegionValue>(V))
- return RegionV->getType();
+ if (isa<VPIRValue, VPRegionValue, VPSymbolicValue, VPStandaloneRecipeValue,
+ VPExpandSCEVRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe,
+ VPScalarIVStepsRecipe, VPWidenCanonicalIVRecipe, VPWidenCastRecipe,
+ VPWidenIntrinsicRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
+ VPVectorEndPointerRecipe, VPWidenCallRecipe>(V)) {
+ Type *Ty = V->getScalarType();
+ assert(Ty && "Scalar type must be set by recipe construction");
+ return Ty;
+ }
Type *ResultTy =
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
@@ -281,26 +271,11 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
})
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
- .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
- VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
- VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe>(
- [this](const VPRecipeBase *R) {
- return inferScalarType(R->getOperand(0));
- })
// VPInstructionWithType must be handled before VPInstruction.
- .Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
- VPWidenCastRecipe>(
+ .Case<VPInstructionWithType, VPWidenCastRecipe>(
[](const auto *R) { return R->getResultType(); })
- .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCallRecipe, VPWidenMemoryRecipe>(
+ .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
- .Case([V](const VPInterleaveBase *R) {
- // TODO: Use info from interleave group.
- return V->getUnderlyingValue()->getType();
- })
- .Case([](const VPExpandSCEVRecipe *R) {
- return R->getSCEV()->getType();
- })
.Case([this](const VPReductionRecipe *R) {
return inferScalarType(R->getChainOp());
})
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index b73636c31f0a7..ca5e0c586066b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -24,8 +24,6 @@ class VPValue;
class VPBlendRecipe;
class VPInstruction;
class VPWidenRecipe;
-class VPWidenCallRecipe;
-class VPWidenMemoryRecipe;
class VPReplicateRecipe;
class VPRecipeBase;
class VPlan;
@@ -50,9 +48,7 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
Type *inferScalarTypeForRecipe(const VPInstruction *R);
- Type *inferScalarTypeForRecipe(const VPWidenCallRecipe *R);
Type *inferScalarTypeForRecipe(const VPWidenRecipe *R);
- Type *inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R);
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b66e7225f1743..4a2dfa0f928b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -445,6 +445,20 @@ template class VPUnrollPartAccessor<2>;
template class VPUnrollPartAccessor<3>;
}
+Type *llvm::getScalarTypeOrInfer(VPValue *V) {
+ if (Type *Ty = V->getScalarType())
+ return Ty;
+ auto *Recipe = V->getDefiningRecipe();
+ assert(Recipe && Recipe->getParent() &&
+ "operand without scalar type must be a recipe in a plan");
+ VPTypeAnalysis TypeInfo(*Recipe->getParent()->getPlan());
+ return TypeInfo.inferScalarType(V);
+}
+
+VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr)
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}, Expr->getType()),
+ Expr(Expr) {}
+
VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD,
DebugLoc DL, const Twine &Name)
@@ -2033,7 +2047,7 @@ bool VPWidenIntrinsicRecipe::usesFirstLaneOnly(const VPValue *Op) const {
void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN-INTRINSIC ";
- if (ResultTy->isVoidTy()) {
+ if (getResultType()->isVoidTy()) {
O << "void ";
} else {
printAsOperand(O, SlotTracker);
@@ -2974,7 +2988,7 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
- : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}, {}),
+ : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dadd9253ad57..37d5f6c705737 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2076,16 +2076,17 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
continue;
// Update IV operands and comparison bound to use new narrower type.
- auto *NewStart = Plan.getZero(NewIVTy);
- WideIV->setStartValue(NewStart);
- auto *NewStep = Plan.getConstantInt(NewIVTy, 1);
- WideIV->setStepValue(NewStep);
+ auto *NewWideIV = WideIV->cloneWithOperands(Plan.getZero(NewIVTy),
+ Plan.getConstantInt(NewIVTy, 1),
+ WideIV->getVFValue());
+ NewWideIV->insertBefore(WideIV);
+ WideIV->replaceAllUsesWith(NewWideIV);
auto *NewBTC = new VPWidenCastRecipe(
Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
nullptr, VPIRFlags::getDefaultFlags(Instruction::Trunc));
Plan.getVectorPreheader()->appendRecipe(NewBTC);
- auto *Cmp = cast<VPInstruction>(WideIV->getSingleUser());
+ auto *Cmp = cast<VPInstruction>(NewWideIV->getSingleUser());
Cmp->setOperand(1, NewBTC);
MadeChange = true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index c7e2b299fe48a..0303cc9db7a54 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -194,6 +194,10 @@ class LLVM_ABI_FOR_TEST VPValue {
VPRecipeBase *getDefiningRecipe();
const VPRecipeBase *getDefiningRecipe() const;
+ /// Returns the scalar type of this VPValue, dispatching based on the
+ /// concrete subclass.
+ Type *getScalarType() const;
+
/// Returns true if this VPValue is defined by a recipe.
bool hasDefiningRecipe() const { return getDefiningRecipe(); }
@@ -313,6 +317,9 @@ class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
+ /// The scalar type of the value produced by this recipe.
+ Type *Ty = nullptr;
+
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -321,11 +328,15 @@ class VPRecipeValue : public VPValue {
#endif
protected:
- VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr, Type *Ty = nullptr)
+ : VPValue(SC, UV), Ty(Ty) {}
public:
LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+ /// Returns the scalar type of this VPRecipeValue.
+ Type *getScalarType() const { return Ty; }
+
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
V->getVPValueID() == VPVSingleDefValueSC;
@@ -340,7 +351,7 @@ class VPSingleDefValue : public VPRecipeValue {
protected:
/// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
- Value *UV = nullptr);
+ Value *UV = nullptr, Type *Ty = nullptr);
public:
static bool classof(const VPValue *V) {
@@ -357,7 +368,8 @@ class VPStandaloneRecipeValue : public VPRecipeValue {
public:
LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
- Value *UV = nullptr);
+ Value *UV = nullptr,
+ Type *Ty = nullptr);
~VPStandaloneRecipeValue() override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 4b99829a21817..9396ea95e1922 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -42,6 +42,12 @@ class VPlanVerifier {
/// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
+ /// Verify that the stored scalar type of \p R is consistent with the types
+ /// derived from its operands. A null stored type is tolerated during the
+ /// transition to fully threaded scalar types; once set, it must agree with
+ /// the operand-derived type.
+ bool verifyRecipeTypes(const VPRecipeBase &R) const;
+
bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
bool verifyBlock(const VPBlockBase *VPB);
@@ -199,6 +205,54 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
return true;
}
+bool VPlanVerifier::verifyRecipeTypes(const VPRecipeBase &R) const {
+ const auto *SR = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!SR)
+ return true;
+
+ auto CheckScalarType = [&](Type *Derived) -> bool {
+ if (Derived == SR->getScalarType())
+ return true;
+ errs() << "Recipe result type does not match type derived from operands";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ errs() << ": ";
+ R.dump();
+#endif
+ errs() << "\n";
+ return false;
+ };
+
+ auto CheckOperandTypes = [&]() -> bool {
+ if (all_of(drop_begin(R.operands()), [&R](VPValue *Op) {
+ return getScalarTypeOrInfer(R.getOperand(0)) ==
+ getScalarTypeOrInfer(Op);
+ }))
+ return true;
+ errs() << "Recipe operand types do not match";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ errs() << ": ";
+ R.dump();
+#endif
+ errs() << "\n";
+ return false;
+ };
+
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
+ case VPRecipeBase::VPWidenPHISC:
+ case VPRecipeBase::VPPredInstPHISC:
+ return CheckOperandTypes() &&
+ CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
+ default:
+ return true;
+ }
+ llvm_unreachable("all recipes must be handled above");
+}
+
bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyPhiRecipes(VPBB))
return false;
@@ -219,6 +273,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
errs() << "not in a VPIRBasicBlock!\n";
return false;
}
+ if (!verifyRecipeTypes(R))
+ return false;
for (const VPValue *V : R.definedValues()) {
// Verify that we can infer a scalar type for each defined value. With
// assertions enabled, inferScalarType will perform some consistency
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 9cc1d90d6256d..0dbee19675a54 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -721,10 +721,9 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
VPBlockUtils::connectBlocks(VPBB1, VPBB2);
- auto *WidenPhi = new VPWidenPHIRecipe({});
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- WidenPhi->addOperand(Val);
+ auto *WidenPhi = new VPWidenPHIRecipe(ArrayRef<VPValue *>{Val});
VPBB2->appendRecipe(WidenPhi);
VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
@@ -743,11 +742,9 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
- auto *WidenPhi = new VPWidenPHIRecipe({});
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- WidenPhi->addOperand(Val);
- WidenPhi->addOperand(Val);
+ auto *WidenPhi = new VPWidenPHIRecipe({Val, Val});
VPBB2->appendRecipe(WidenPhi);
VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index de084ec9a070a..37c8f1f0d534a 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -330,6 +330,44 @@ TEST_F(VPVerifierTest, NonHeaderPHIInHeader) {
delete PHINode;
}
+TEST_F(VPVerifierTest, VPScalarIVStepsMismatchedStepType) {
+ VPlan &Plan = getPlan();
+ VPIRValue *IV = Plan.getConstantInt(32, 0);
+ // Step has a different scalar type than the IV; the verifier must reject.
+ VPIRValue *Step = Plan.getConstantInt(64, 1);
+ VPIRValue *VF = Plan.getConstantInt(32, 4);
+
+ auto *Steps = new VPScalarIVStepsRecipe(IV, Step, VF, Instruction::Add,
+ FastMathFlags(), DebugLoc());
+ VPInstruction *BranchOnCond =
+ new VPInstruction(VPInstruction::BranchOnCond, {IV});
+
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBB2->appendRecipe(Steps);
+ VPBB2->appendRecipe(BranchOnCond);
+
+ VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
+ "R1", VPBB2, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, R1);
+ VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
+
+#if GTEST_HAS_STREAM_REDIRECTION
+ ::testing::internal::CaptureStderr();
+#endif
+ EXPECT_FALSE(verifyVPlanIsValid(Plan));
+#if GTEST_HAS_STREAM_REDIRECTION
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ EXPECT_STREQ("Recipe result type does not match type derived from operands: "
+ "vp<%2> = SCALAR-STEPS ir<0>, ir<1>, ir<4>\n\n",
+ ::testing::internal::GetCapturedStderr().c_str());
+#else
+ EXPECT_STREQ("Recipe result type does not match type derived from operands\n",
+ ::testing::internal::GetCapturedStderr().c_str());
+#endif
+#endif
+}
+
TEST_F(VPVerifierTest, testRUN_VPLAN_PASS) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
>From 0b2999c567c6b5e12e2988050d17494c0fa5576c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 3 May 2026 20:26:07 +0100
Subject: [PATCH 4/4] [VPlan] Assert that replacement types match in
VPUser::setOperand (NFC).
Add assertion to VPValue::setOperand to check if types of the new
operand matches the old operand.
This makes it easier to catch replacements with incorrect types at the
source, instead only later during verification.
A few places currently preform replacements with mis-matching types,
which only get fixed up later. Update those to avoid type-violation.
Depends on https://github.com/llvm/llvm-project/pull/195485 (included in
PR)
---
llvm/lib/Transforms/Vectorize/VPlan.h | 10 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 133 ++++++++++++------
llvm/lib/Transforms/Vectorize/VPlanValue.h | 3 +
.../RISCV/evl-compatible-loops.ll | 2 +-
.../LoopVectorize/RISCV/induction-costs.ll | 4 +-
.../LoopVectorize/RISCV/iv-select-cmp.ll | 2 +-
.../LoopVectorize/RISCV/pointer-induction.ll | 2 +-
.../RISCV/riscv-vector-reverse.ll | 12 +-
.../LoopVectorize/RISCV/strided-accesses.ll | 8 +-
.../RISCV/tail-folding-cond-reduction.ll | 8 +-
...an-vp-intrinsics-fixed-order-recurrence.ll | 9 +-
.../RISCV/vplan-vp-intrinsics-reduction.ll | 18 +--
.../VPlan/RISCV/vplan-vp-intrinsics.ll | 9 +-
13 files changed, 135 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a4c6159b2cdd3..1e5c57874f477 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1814,11 +1814,13 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
~VPWidenRecipe() override = default;
- VPWidenRecipe *clone() override {
+ VPWidenRecipe *clone() override { return cloneWithOperands(operands()); }
+
+ VPWidenRecipe *cloneWithOperands(ArrayRef<VPValue *> NewOperands) {
if (auto *UV = getUnderlyingValue())
- return new VPWidenRecipe(*cast<Instruction>(UV), operands(), *this, *this,
- getDebugLoc());
- return new VPWidenRecipe(Opcode, operands(), *this, *this, getDebugLoc());
+ return new VPWidenRecipe(*cast<Instruction>(UV), NewOperands, *this,
+ *this, getDebugLoc());
+ return new VPWidenRecipe(Opcode, NewOperands, *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 37d5f6c705737..42ca70aca97ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2575,6 +2575,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
VPTypeAnalysis TypeInfo(Plan);
VPBasicBlock *PH = Plan.getVectorPreheader();
+ SmallVector<VPRecipeBase *> ToErase;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -2608,54 +2609,72 @@ void VPlanTransforms::truncateToMinimalBitwidths(
if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(&R))
VPW->dropPoisonGeneratingFlags();
- if (OldResSizeInBits != NewResSizeInBits &&
- !match(&R, m_ICmp(m_VPValue(), m_VPValue()))) {
- // Extend result to original width.
+ assert((OldResSizeInBits != NewResSizeInBits ||
+ match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
+ "Only ICmps should not need extending the result.");
+ assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
+
+ // Insert a ZExt of \p Source's result back to OldResTy after \p After,
+ // so wide users keep seeing the original wide type.
+ auto InsertExtAfter = [&](VPValue *Source,
+ VPRecipeBase *After) -> VPWidenCastRecipe * {
+ assert(OldResSizeInBits > NewResSizeInBits && "Nothing to shrink?");
auto *Ext = new VPWidenCastRecipe(
- Instruction::ZExt, ResultVPV, OldResTy, nullptr,
+ Instruction::ZExt, Source, OldResTy, nullptr,
VPIRFlags::getDefaultFlags(Instruction::ZExt));
- Ext->insertAfter(&R);
- ResultVPV->replaceAllUsesWith(Ext);
- Ext->setOperand(0, ResultVPV);
- assert(OldResSizeInBits > NewResSizeInBits && "Nothing to shrink?");
- } else {
- assert(match(&R, m_ICmp(m_VPValue(), m_VPValue())) &&
- "Only ICmps should not need extending the result.");
- }
+ Ext->insertAfter(After);
+ return Ext;
+ };
- assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
- if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(&R))
+ // For loads/intrinsics we don't recreate the recipe; just wrap the
+ // original wide result in a ZExt.
+ if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(&R)) {
+ if (OldResSizeInBits != NewResSizeInBits) {
+ auto *Ext = InsertExtAfter(ResultVPV, &R);
+ ResultVPV->replaceAllUsesWith(Ext);
+ Ext->setOperand(0, ResultVPV);
+ }
continue;
+ }
// Shrink operands by introducing truncates as needed.
unsigned StartIdx =
match(&R, m_Select(m_VPValue(), m_VPValue(), m_VPValue())) ? 1 : 0;
- for (unsigned Idx = StartIdx; Idx != R.getNumOperands(); ++Idx) {
- auto *Op = R.getOperand(Idx);
+ SmallVector<VPValue *> NewOperands(R.operands());
+ for (VPValue *&Op : MutableArrayRef(NewOperands).drop_front(StartIdx)) {
unsigned OpSizeInBits =
TypeInfo.inferScalarType(Op)->getScalarSizeInBits();
if (OpSizeInBits == NewResSizeInBits)
continue;
assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
auto [ProcessedIter, IterIsEmpty] = ProcessedTruncs.try_emplace(Op);
- if (!IterIsEmpty) {
- R.setOperand(Idx, ProcessedIter->second);
- continue;
+ if (IterIsEmpty) {
+ VPBuilder Builder;
+ if (isa<VPIRValue>(Op))
+ Builder.setInsertPoint(PH);
+ else
+ Builder.setInsertPoint(&R);
+ ProcessedIter->second =
+ Builder.createWidenCast(Instruction::Trunc, Op, NewResTy);
}
-
- VPBuilder Builder;
- if (isa<VPIRValue>(Op))
- Builder.setInsertPoint(PH);
- else
- Builder.setInsertPoint(&R);
- VPWidenCastRecipe *NewOp =
- Builder.createWidenCast(Instruction::Trunc, Op, NewResTy);
- ProcessedIter->second = NewOp;
- R.setOperand(Idx, NewOp);
+ Op = ProcessedIter->second;
}
+ auto *NWR = cast<VPWidenRecipe>(&R)->cloneWithOperands(NewOperands);
+ NWR->insertBefore(&R);
+
+ // Wrap NWR in a ZExt to preserve the original wide type for downstream
+ // users (unless this is an ICmp, which produces i1 regardless).
+ VPValue *Replacement = NWR->getVPSingleValue();
+ if (OldResSizeInBits != NewResSizeInBits)
+ Replacement = InsertExtAfter(Replacement, NWR)->getVPSingleValue();
+ ResultVPV->replaceAllUsesWith(Replacement);
+ R.removeFromParent();
+ ToErase.push_back(&R);
}
}
+ for (VPRecipeBase *R : ToErase)
+ delete R;
}
void VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
@@ -2897,10 +2916,14 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
VPValue *Addr, *Mask, *EndPtr;
/// Adjust any end pointers so that they point to the end of EVL lanes not VF.
- auto AdjustEndPtr = [&CurRecipe, &EVL](VPValue *EndPtr) {
+ auto AdjustEndPtr = [&CurRecipe, &EVL, &TypeInfo](VPValue *EndPtr) {
auto *EVLEndPtr = cast<VPVectorEndPointerRecipe>(EndPtr)->clone();
EVLEndPtr->insertBefore(&CurRecipe);
- EVLEndPtr->setOperand(1, &EVL);
+ // Cast EVL (i32) to match the VF operand's type.
+ VPValue *EVLAsVF = VPBuilder(EVLEndPtr).createScalarZExtOrTrunc(
+ &EVL, TypeInfo.inferScalarType(EVLEndPtr->getOperand(1)),
+ TypeInfo.inferScalarType(&EVL), DebugLoc::getUnknown());
+ EVLEndPtr->setOperand(1, EVLAsVF);
return EVLEndPtr;
};
@@ -3058,11 +3081,23 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+ // EVL is i32 but VF/VFxUF are IdxTy. Materialize an IdxTy-typed EVL right
+ // after EVL so replacements preserve operand scalar types.
+ Type *IdxTy = TypeInfo.inferScalarType(&Plan.getVF());
+ Type *EVLTy = TypeInfo.inferScalarType(&EVL);
+ VPValue *EVLAsIdx = &EVL;
+ if (EVLTy != IdxTy) {
+ VPRecipeBase *EVLR = EVL.getDefiningRecipe();
+ VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
+ EVLAsIdx = Builder.createScalarZExtOrTrunc(&EVL, IdxTy, EVLTy,
+ DebugLoc::getUnknown());
+ }
+
assert(all_of(Plan.getVF().users(),
IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
VPWidenIntOrFpInductionRecipe>) &&
"User of VF that we can't transform to EVL.");
- Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
+ Plan.getVF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
});
@@ -3073,7 +3108,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
m_Isa<VPWidenPointerInductionRecipe>()))) &&
"Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
"increment of the canonical induction.");
- Plan.getVFxUF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
+ Plan.getVFxUF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
// Only replace uses in VPWidenPointerInductionRecipe; The increment of the
// canonical induction must not be updated.
return isa<VPWidenPointerInductionRecipe>(U);
@@ -4500,10 +4535,12 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
*Ext1, Ext1->getDebugLoc());
NewExt1->insertBefore(Ext1);
}
- Mul->setOperand(0, NewExt0);
- Mul->setOperand(1, NewExt1);
- Red->setOperand(1, Mul);
- return new VPExpressionRecipe(NewExt0, NewExt1, Mul, Red);
+ auto *NewMul = Mul->cloneWithOperands({NewExt0, NewExt1});
+ NewMul->insertBefore(Mul);
+ Ext->replaceAllUsesWith(NewMul);
+ Ext->eraseFromParent();
+ Mul->eraseFromParent();
+ return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
}
}
return nullptr;
@@ -5962,15 +5999,19 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op,
MulLHS->getOpcode() != MulRHS->getOpcode())
return Op;
VPBuilder Builder(Mul);
- Mul->setOperand(0, Builder.createWidenCast(MulLHS->getOpcode(),
- MulLHS->getOperand(0),
- Ext->getResultType()));
- Mul->setOperand(1, MulLHS == MulRHS
- ? Mul->getOperand(0)
- : Builder.createWidenCast(MulRHS->getOpcode(),
- MulRHS->getOperand(0),
- Ext->getResultType()));
- return Mul;
+ auto *NewLHS = Builder.createWidenCast(
+ MulLHS->getOpcode(), MulLHS->getOperand(0), Ext->getResultType());
+ auto *NewRHS = MulLHS == MulRHS
+ ? NewLHS
+ : Builder.createWidenCast(MulRHS->getOpcode(),
+ MulRHS->getOperand(0),
+ Ext->getResultType());
+ auto *NewMul = Mul->cloneWithOperands({NewLHS, NewRHS});
+ Builder.insert(NewMul);
+ Op->replaceAllUsesWith(NewMul);
+ Op->eraseFromParent();
+ Mul->eraseFromParent();
+ return NewMul;
}
return Op;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 0303cc9db7a54..5dadfd7e30b49 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -427,6 +427,9 @@ class VPUser {
}
void setOperand(unsigned I, VPValue *New) {
+ assert((!Operands[I]->getScalarType() || !New->getScalarType() ||
+ Operands[I]->getScalarType() == New->getScalarType()) &&
+ "scalar type of new operand must match the old operand");
Operands[I]->removeUser(*this);
Operands[I] = New;
New->addUser(*this);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
index 361fb6d15dc60..a557600972109 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
@@ -63,9 +63,9 @@ define void @test_wide_ptr_induction(ptr noalias %a, ptr noalias %b, i64 %N) {
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 2 x i64> [[TMP5]], splat (i64 3)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP6]]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2p0.p0(<vscale x 2 x ptr> [[VECTOR_GEP]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index 66a7493b067c8..1752ea0814fcc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -137,6 +137,7 @@ define void @test_3_inductions(ptr noalias %dst, ptr noalias %src, i64 %n) #1 {
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 2 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
@@ -144,7 +145,6 @@ define void @test_3_inductions(ptr noalias %dst, ptr noalias %src, i64 %n) #1 {
; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[TMP7]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT3]]
; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT3]]
@@ -191,6 +191,7 @@ define void @redundant_iv_trunc_for_cse(ptr noalias %src, ptr noalias %dst, i64
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 4 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[EVL_BASED_IV]]
@@ -201,7 +202,6 @@ define void @redundant_iv_trunc_for_cse(ptr noalias %src, ptr noalias %dst, i64
; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 4 x i32> [[PREDPHI]] to <vscale x 4 x i8>
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[EVL_BASED_IV]]
; CHECK-NEXT: call void @llvm.vp.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP7]], ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
index b8a7fb7bda860..f5458c88f24a2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
@@ -17,6 +17,7 @@ define i32 @find_last_trunc_iv(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP1]], %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]
@@ -27,7 +28,6 @@ define i32 @find_last_trunc_iv(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP10]] = select i1 [[TMP8]], <vscale x 4 x i32> [[VEC_IND]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP11]], [[TMP22]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 786ef735fc7ad..103488c316c8c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -97,13 +97,13 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 12)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 4
; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
-; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP27]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index e5f42b0d41682..0af1762da3bae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -62,6 +62,7 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
; RV32-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
; RV32-NEXT: [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
; RV32-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP7]]
+; RV32-NEXT: [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
; RV32-NEXT: [[TMP3:%.*]] = sub nuw nsw i32 [[TMP9]], 1
; RV32-NEXT: [[TMP4:%.*]] = sub i32 0, [[TMP3]]
; RV32-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP8]], i32 [[TMP4]]
@@ -72,7 +73,6 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
; RV32-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[TMP16]], i32 [[TMP4]]
; RV32-NEXT: [[VP_REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; RV32-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE1]], ptr align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; RV32-NEXT: [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]]
; RV32-NEXT: [[TMP21:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -240,6 +240,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: [[TMP13:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
; RV32-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
; RV32-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP14]]
+; RV32-NEXT: [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
; RV32-NEXT: [[TMP9:%.*]] = sub nuw nsw i32 [[TMP16]], 1
; RV32-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]]
; RV32-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP15]], i32 [[TMP10]]
@@ -250,7 +251,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP23]], i32 [[TMP10]]
; RV32-NEXT: [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; RV32-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
-; RV32-NEXT: [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]]
; RV32-NEXT: [[TMP30:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -337,7 +337,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; RV64-UF2: [[SCALAR_PH]]:
; RV64-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV64-UF2-NEXT: [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV64-UF2-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
; RV64-UF2-NEXT: br label %[[FOR_BODY:.*]]
; RV64-UF2: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; RV64-UF2-NEXT: br label %[[FOR_COND_CLEANUP]]
@@ -462,6 +462,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: [[TMP13:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
; RV32-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
; RV32-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]]
+; RV32-NEXT: [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
; RV32-NEXT: [[TMP9:%.*]] = sub nuw nsw i32 [[TMP16]], 1
; RV32-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]]
; RV32-NEXT: [[TMP28:%.*]] = getelementptr float, ptr [[TMP15]], i32 [[TMP10]]
@@ -472,7 +473,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: [[TMP25:%.*]] = getelementptr float, ptr [[TMP23]], i32 [[TMP10]]
; RV32-NEXT: [[VP_REVERSE3:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; RV32-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE3]], ptr align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
-; RV32-NEXT: [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]]
; RV32-NEXT: [[TMP30:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -559,7 +559,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; RV64-UF2: [[SCALAR_PH]]:
; RV64-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV64-UF2-NEXT: [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV64-UF2-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
; RV64-UF2-NEXT: br label %[[FOR_BODY:.*]]
; RV64-UF2: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; RV64-UF2-NEXT: br label %[[FOR_COND_CLEANUP]]
@@ -640,6 +640,7 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
; RV32-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
; RV32-NEXT: [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
; RV32-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]]
+; RV32-NEXT: [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
; RV32-NEXT: [[TMP3:%.*]] = sub nuw nsw i32 [[TMP9]], 1
; RV32-NEXT: [[TMP4:%.*]] = sub i32 0, [[TMP3]]
; RV32-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[TMP8]], i32 [[TMP4]]
@@ -650,7 +651,6 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
; RV32-NEXT: [[TMP22:%.*]] = getelementptr float, ptr [[TMP16]], i32 [[TMP4]]
; RV32-NEXT: [[VP_REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
; RV32-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE1]], ptr align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; RV32-NEXT: [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]]
; RV32-NEXT: [[TMP21:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 0cff922ba09a3..c9ee76349c09b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -228,10 +228,10 @@ define void @single_constant_stride_ptr_iv(ptr %p) {
; CHECK-NEXT: [[TMP16:%.*]] = shl <vscale x 4 x i64> [[TMP14]], splat (i64 3)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP16]]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP9]], 3
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
@@ -1150,10 +1150,10 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI11]], <vscale x 4 x i64> [[TMP18]]
; STRIDED-NEXT: [[VECTOR_GEP7:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP18]]
; STRIDED-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[VECTOR_GEP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META12:![0-9]+]]
-; STRIDED-NEXT: [[TMP30:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
-; STRIDED-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP30]], <vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META15:![0-9]+]], !noalias [[META12]]
; STRIDED-NEXT: [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[VECTOR_GEP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META12:![0-9]+]]
+; STRIDED-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
+; STRIDED-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP9]], <vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META15:![0-9]+]], !noalias [[META12]]
; STRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
; STRIDED-NEXT: [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP16]]
; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP25]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
index c2ac80c11ce81..2aff016883ae5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
@@ -387,6 +387,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP10]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-OUTLOOP-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT: [[TMP20:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-OUTLOOP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
; IF-EVL-OUTLOOP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-OUTLOOP-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
@@ -395,7 +396,6 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: [[TMP17:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
; IF-EVL-OUTLOOP-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
; IF-EVL-OUTLOOP-NEXT: [[TMP19]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP12]])
-; IF-EVL-OUTLOOP-NEXT: [[TMP20:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT1]] = add i64 [[TMP20]], [[EVL_BASED_IV1]]
; IF-EVL-OUTLOOP-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
; IF-EVL-OUTLOOP-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
@@ -420,6 +420,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-INLOOP-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-INLOOP-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT: [[TMP19:%.*]] = zext i32 [[TMP11]] to i64
; IF-EVL-INLOOP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
; IF-EVL-INLOOP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-INLOOP-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
@@ -428,7 +429,6 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
; IF-EVL-INLOOP-NEXT: [[TMP17:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-INLOOP-NEXT: [[ADD]] = add i32 [[TMP17]], [[RDX]]
-; IF-EVL-INLOOP-NEXT: [[TMP19:%.*]] = zext i32 [[TMP11]] to i64
; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT1]] = add i64 [[TMP19]], [[EVL_BASED_IV1]]
; IF-EVL-INLOOP-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
; IF-EVL-INLOOP-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
@@ -583,6 +583,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[TMP10]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT7:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-OUTLOOP-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-OUTLOOP-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT: [[TMP25:%.*]] = zext i32 [[TMP14]] to i64
; IF-EVL-OUTLOOP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
; IF-EVL-OUTLOOP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-OUTLOOP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
@@ -591,7 +592,6 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: [[TMP22:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]
; IF-EVL-OUTLOOP-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[VEC_PHI]]
; IF-EVL-OUTLOOP-NEXT: [[TMP24]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
-; IF-EVL-OUTLOOP-NEXT: [[TMP25:%.*]] = zext i32 [[TMP14]] to i64
; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP25]], [[IV]]
; IF-EVL-OUTLOOP-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP25]]
; IF-EVL-OUTLOOP-NEXT: [[VEC_IND_NEXT7]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT2]]
@@ -616,6 +616,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-INLOOP-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-INLOOP-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
; IF-EVL-INLOOP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
; IF-EVL-INLOOP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-INLOOP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
@@ -623,7 +624,6 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: [[TMP15:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
; IF-EVL-INLOOP-NEXT: [[TMP16:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> [[TMP15]], i32 [[TMP11]])
; IF-EVL-INLOOP-NEXT: [[TMP17]] = add i32 [[TMP16]], [[VEC_PHI]]
-; IF-EVL-INLOOP-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[IV]]
; IF-EVL-INLOOP-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP18]]
; IF-EVL-INLOOP-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index bfde3ada849cd..7cdc33a44b3a6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -28,7 +28,8 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%TC>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<[[PREV_EVL:%.+]]> = phi [ vp<[[VF32]]>, vector.ph ], [ vp<[[EVL:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<[[EVL]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[CAST]]>
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds nuw ir<%A>, vp<[[ST]]
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds nuw ir<[[GEP1]]>
; IF-EVL-NEXT: WIDEN ir<[[LD]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -37,9 +38,9 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds nuw ir<%B>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds nuw ir<[[GEP2]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ADD]]>, vp<[[EVL]]>
-; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
-; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
-; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST2:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST2]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST2]]>
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
; IF-EVL-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
index 99970c03d0531..46ef47c957a66 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -41,15 +41,16 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
+; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[CAST]]>
; IF-EVL-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = add ir<[[LD1]]>, ir<[[RDX_PHI]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[RDX_SELECT]]> = call llvm.vp.merge(ir<true>, ir<[[ADD]]>, ir<[[RDX_PHI]]>, vp<[[EVL]]>)
-; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
-; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
-; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
+; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[CAST2:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST2]]>, vp<[[EVL_PHI]]>
+; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST2]]>
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
; IF-EVL-OUTLOOP-NEXT: No successors
@@ -82,14 +83,15 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
+; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[CAST]]>
; IF-EVL-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
; IF-EVL-INLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
; IF-EVL-INLOOP-NEXT: REDUCE ir<[[ADD:%.+]]> = ir<[[RDX_PHI]]> + vp.reduce.add (ir<[[LD1]]>, vp<[[EVL]]>)
-; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
-; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
-; IF-EVL-INLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
+; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[CAST2:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST2]]>, vp<[[EVL_PHI]]>
+; IF-EVL-INLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST2]]>
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
; IF-EVL-INLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
; IF-EVL-INLOOP-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
index 510d17ab16bee..1a185cae77624 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
@@ -25,7 +25,8 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: CURRENT-ITERATION-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%N>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<%evl>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<%evl> to i64
+; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[CAST]]>
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<%evl>
@@ -36,9 +37,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<%evl>
-; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<%evl> to i64
-; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
-; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST2:%[0-9]+]]> = zext vp<%evl> to i64
+; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST2]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST2]]>
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
; IF-EVL-NEXT: No successors
More information about the llvm-commits
mailing list