[llvm] 3eb5301 - [VPlan] Make VPRegionValue a subclass of VPSymbolicValue. (NFC) (#196753)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 4 13:45:35 PDT 2026
Author: Florian Hahn
Date: 2026-07-04T21:45:30+01:00
New Revision: 3eb53013dca38fc0fa0c0e11de9fef1a7a14b9c7
URL: https://github.com/llvm/llvm-project/commit/3eb53013dca38fc0fa0c0e11de9fef1a7a14b9c7
DIFF: https://github.com/llvm/llvm-project/commit/3eb53013dca38fc0fa0c0e11de9fef1a7a14b9c7.diff
LOG: [VPlan] Make VPRegionValue a subclass of VPSymbolicValue. (NFC) (#196753)
Turn VPRegionValue into a subclass of VPSymbolicValue, ensuring we also
guard against premature RAUW of region values.
Split off from approved https://github.com/llvm/llvm-project/pull/196199
PR: https://github.com/llvm/llvm-project/pull/196753
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanValue.h
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9c0bec2350c97..1e55f2834ef8c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1234,17 +1234,19 @@ VPlan *VPlan::duplicate() {
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
+ assert(none_of(Old2NewVPValues.keys(), IsaPred<VPSymbolicValue>) &&
+ "All VPSymbolicValues must be handled below");
+
if (auto *LoopRegion = getVectorLoopRegion()) {
auto *OldCanIV = LoopRegion->getCanonicalIV();
auto *NewCanIV = NewPlan->getVectorLoopRegion()->getCanonicalIV();
assert(OldCanIV && NewCanIV &&
"Loop regions of both plans must have canonical IVs.");
Old2NewVPValues[OldCanIV] = NewCanIV;
+ if (OldCanIV->isMaterialized())
+ NewCanIV->markMaterialized();
}
- assert(none_of(Old2NewVPValues.keys(), IsaPred<VPSymbolicValue>) &&
- "All VPSymbolicValues must be handled below");
-
if (BackedgeTakenCount)
NewPlan->BackedgeTakenCount =
new VPSymbolicValue(BackedgeTakenCount->getType());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 0815ea2c54333..c57e106021637 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1081,6 +1081,16 @@ inline auto m_WidenIntrinsic(const T &...Ops) {
return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
}
+/// Match VPValues that represent live-ins: VPIRValues and (plain)
+/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
+/// not live-ins and are excluded.
+struct LiveIn_match {
+ template <typename ITy> bool match(ITy *V) const {
+ return isa<VPIRValue>(V) ||
+ (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
+ }
+};
+
inline VPInstruction_match<VPInstruction::VScale> m_VScale() {
return m_VPInstruction<VPInstruction::VScale>();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bd260a0d12362..39e8ae5dd3b8c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3468,8 +3468,10 @@ void VPlanTransforms::addExplicitVectorLength(
// Replace all uses of the canonical IV with VPCurrentIterationPHIRecipe
// except for the canonical IV increment.
- CanonicalIV->replaceAllUsesWith(CurrentIteration);
- CanonicalIVIncrement->setOperand(0, CanonicalIV);
+ CanonicalIV->replaceUsesWithIf(CurrentIteration,
+ [CanonicalIVIncrement](VPUser &U, unsigned) {
+ return &U != CanonicalIVIncrement;
+ });
// TODO: support unroll factor > 1.
Plan.setUF(1);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index bcd17a54a3e31..594c201d5a8cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -78,7 +78,7 @@ class UnrollState {
void unrollBlock(VPBlockBase *VPB);
VPValue *getValueForPart(VPValue *V, unsigned Part) {
- if (Part == 0 || isa<VPIRValue, VPSymbolicValue, VPRegionValue>(V))
+ if (Part == 0 || isa<VPIRValue, VPSymbolicValue>(V))
return V;
assert((VPV2Parts.contains(V) && VPV2Parts[V].size() >= Part) &&
"accessed value does not exist");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f19e7538e9c73..099f273dd0f95 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -171,13 +171,6 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
PredicatedScalarEvolution &PSE,
const Loop *L) {
ScalarEvolution &SE = *PSE.getSE();
- if (isa<VPIRValue, VPSymbolicValue>(V)) {
- Value *LiveIn = V->getUnderlyingValue();
- if (LiveIn && SE.isSCEVable(LiveIn->getType()))
- return SE.getSCEV(LiveIn);
- return SE.getCouldNotCompute();
- }
-
if (auto *RV = dyn_cast<VPRegionValue>(V)) {
assert(RV == RV->getDefiningRegion()->getCanonicalIV() &&
"RegionValue must be canonical IV");
@@ -187,6 +180,13 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
L, SCEV::FlagAnyWrap);
}
+ if (isa<VPIRValue, VPSymbolicValue>(V)) {
+ Value *LiveIn = V->getUnderlyingValue();
+ if (LiveIn && SE.isSCEVable(LiveIn->getType()))
+ return SE.getSCEV(LiveIn);
+ return SE.getCouldNotCompute();
+ }
+
// Helper to create SCEVs for binary and unary operations.
auto CreateSCEV = [&](ArrayRef<VPValue *> Ops,
function_ref<const SCEV *(ArrayRef<SCEVUse>)> CreateFn)
@@ -421,7 +421,7 @@ static bool preservesUniformity(unsigned Opcode) {
bool vputils::isSingleScalar(const VPValue *VPV) {
// Live-in, symbolic and region-values represent single-scalar values.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
+ if (isa<VPIRValue, VPSymbolicValue>(VPV))
return true;
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
@@ -458,7 +458,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
// Live-ins and region values are uniform.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(V))
+ if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
const VPRecipeBase *R = V->getDefiningRecipe();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a7aa0523ad5d0..49b612ea41fca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -49,7 +49,7 @@ class VPSingleDefRecipe;
/// coming from the input IR, symbolic values and values defined by recipes.
class LLVM_ABI_FOR_TEST VPValue {
friend struct VPIRValue;
- friend struct VPSymbolicValue;
+ friend class VPSymbolicValue;
friend class VPRecipeValue;
friend class VPRegionValue;
@@ -212,15 +212,48 @@ class LLVM_ABI_FOR_TEST VPValue {
}
};
+/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
+/// VFxUF.
+class VPSymbolicValue : public VPValue {
+ /// The scalar type of this symbolic value.
+ Type *Ty;
+
+ /// Track whether this value has been materialized (replaced). After
+ /// materialization, accessing users should trigger an assertion.
+ bool Materialized = false;
+
+protected:
+ VPSymbolicValue(unsigned char SC, Type *Ty) : VPValue(SC, nullptr), Ty(Ty) {}
+
+public:
+ VPSymbolicValue(Type *Ty) : VPSymbolicValue(VPVSymbolicSC, Ty) {}
+
+ /// Returns the scalar type of this symbolic value.
+ Type *getType() const { return Ty; }
+
+ /// Returns true if this value has been materialized.
+ bool isMaterialized() const { return Materialized; }
+
+ /// Mark this value as materialized.
+ void markMaterialized() {
+ assert(!Materialized && "VPSymbolicValue already materialized");
+ Materialized = true;
+ }
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSymbolicSC ||
+ V->getVPValueID() == VPRegionValueSC;
+ }
+};
+
/// VPValues defined by a VPRegionBlock, like the canonical IV.
-class VPRegionValue : public VPValue {
+class VPRegionValue : public VPSymbolicValue {
VPRegionBlock *DefiningRegion;
- Type *Ty;
DebugLoc DL;
public:
VPRegionValue(Type *Ty, DebugLoc DL, VPRegionBlock *Region)
- : VPValue(VPValue::VPRegionValueSC), DefiningRegion(Region), Ty(Ty),
+ : VPSymbolicValue(VPValue::VPRegionValueSC, Ty), DefiningRegion(Region),
DL(DL) {}
~VPRegionValue() override = default;
@@ -228,9 +261,6 @@ class VPRegionValue : public VPValue {
/// Returns the region that defines this value.
VPRegionBlock *getDefiningRegion() const { return DefiningRegion; }
- /// Returns the type of the VPRegionValue.
- Type *getType() const { return Ty; }
-
/// Returns the debug location of the VPRegionValue.
DebugLoc getDebugLoc() const { return DL; }
@@ -282,36 +312,6 @@ struct VPConstantInt : public VPIRValue {
uint64_t getZExtValue() const { return getAPInt().getZExtValue(); }
};
-/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
-/// VFxUF.
-struct VPSymbolicValue : public VPValue {
- VPSymbolicValue(Type *Ty) : VPValue(VPVSymbolicSC, nullptr), Ty(Ty) {}
-
- static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVSymbolicSC;
- }
-
- /// Returns the scalar type of this symbolic value.
- Type *getType() const { return Ty; }
-
- /// Returns true if this symbolic value has been materialized.
- bool isMaterialized() const { return Materialized; }
-
- /// Mark this symbolic value as materialized.
- void markMaterialized() {
- assert(!Materialized && "VPSymbolicValue already materialized");
- Materialized = true;
- }
-
-private:
- /// The scalar type of this symbolic value.
- Type *Ty;
-
- /// Track whether this symbolic value has been materialized (replaced).
- /// After materialization, accessing users should trigger an assertion.
- bool Materialized = false;
-};
-
/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 96fec844820d8..7c49487dac459 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1894,6 +1894,32 @@ TEST_F(VPUtilsTest, IsUniformAcrossVFsAndUFsForSingleScalarOpcodes) {
vputils::isUniformAcrossVFsAndUFs(FirstActiveLaneNonUniform.get()));
}
+TEST_F(VPBasicBlockTest, VPRegionValueClonePropagatesMaterialized) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *Preheader = Plan.getEntry();
+ VPBasicBlock *Header = Plan.createVPBasicBlock("header");
+ VPBasicBlock *Latch = Plan.createVPBasicBlock("latch");
+ VPRegionBlock *Region = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
+ "loop", Header, Latch);
+ VPBlockUtils::connectBlocks(Header, Latch);
+ VPBlockUtils::connectBlocks(Preheader, Region);
+ VPBlockUtils::connectBlocks(Region, Plan.getScalarHeader());
+
+ VPRegionValue *CanIV = Region->getCanonicalIV();
+ EXPECT_TRUE(isa<VPSymbolicValue>(CanIV));
+ EXPECT_FALSE(CanIV->isMaterialized());
+
+ // Materialize the canonical IV by replacing all uses, then verify clone
+ // propagates the materialized state.
+ CanIV->replaceAllUsesWith(Plan.getConstantInt(32, 0));
+ EXPECT_TRUE(CanIV->isMaterialized());
+
+ std::unique_ptr<VPlan> Clone(Plan.duplicate());
+ VPRegionValue *ClonedCanIV = Clone->getVectorLoopRegion()->getCanonicalIV();
+ EXPECT_NE(CanIV, ClonedCanIV);
+ EXPECT_TRUE(ClonedCanIV->isMaterialized());
+}
+
#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
TEST_F(VPInstructionTest, VPSymbolicValueConstructUserAfterMaterialization) {
VPlan &Plan = getPlan();
More information about the llvm-commits
mailing list