[llvm] [VPlan] Support live-ins without underlying IR in type analysis. (PR #80723)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 21 10:41:57 PST 2024
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/80723
>From 15b2c7c33347a9876093d8fb56451b55bf3e043c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 5 Feb 2024 18:40:33 +0000
Subject: [PATCH 1/2] [VPlan] Support live-ins without underlying IR in type
analysis.
A VPlan contains multiple live-ins without underlying IR, like VFxUF or
VectorTripCount. Trying to infer the scalar type of those causes a crash
at the moment.
Update VPTypeAnalysis to take a VPlan in its constructor and assign
types to those live-ins up front. All those live-ins share the type of
the canonical IV.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 5 ++++-
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 17 +++++++++++------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 3 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++----
4 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 162a3c4b195e53..43f10c315f2e98 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -238,7 +238,7 @@ struct VPTransformState {
DominatorTree *DT, IRBuilderBase &Builder,
InnerLoopVectorizer *ILV, VPlan *Plan, LLVMContext &Ctx)
: VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan),
- LVer(nullptr), TypeAnalysis(Ctx) {}
+ LVer(nullptr), TypeAnalysis(*Plan, Ctx) {}
/// The chosen Vectorization and Unroll Factors of the loop being vectorized.
ElementCount VF;
@@ -2932,6 +2932,9 @@ class VPlan {
return BackedgeTakenCount;
}
+ /// Return the backedge taken count of the original loop, if set.
+ VPValue *getBackedgeTakenCount() { return BackedgeTakenCount; }
+
/// The vector trip count.
VPValue &getVectorTripCount() { return VectorTripCount; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b9ffe7e5b7af7e..b71bac15463328 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -35,12 +35,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
CachedTypes[OtherV] = ResTy;
return ResTy;
}
- case Instruction::ICmp: {
- // TODO: Check if types for both operands agree. This also requires
- // type-inference for the vector-trip-count, which is missing at the moment.
- Type *ResTy = inferScalarType(R->getOperand(0));
- return ResTy;
- }
+ case Instruction::ICmp:
case VPInstruction::FirstOrderRecurrenceSplice: {
Type *ResTy = inferScalarType(R->getOperand(0));
VPValue *OtherV = R->getOperand(1);
@@ -203,6 +198,16 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
llvm_unreachable("Unhandled opcode");
}
+VPTypeAnalysis::VPTypeAnalysis(VPlan &Plan, LLVMContext &Ctx) : Ctx(Ctx) {
+ auto *CanIV = Plan.getCanonicalIV();
+ Type *CanIVTy = inferScalarType(CanIV);
+ CachedTypes[&Plan.getVectorTripCount()] = CanIVTy;
+ CachedTypes[&Plan.getVFxUF()] = CanIVTy;
+ CachedTypes[Plan.getTripCount()] = CanIVTy;
+ if (auto *BTC = Plan.getBackedgeTakenCount())
+ CachedTypes[BTC] = CanIVTy;
+}
+
Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 7276641551ae80..6b218a39f605c1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -15,6 +15,7 @@ namespace llvm {
class LLVMContext;
class VPValue;
+class VPlan;
class VPBlendRecipe;
class VPInstruction;
class VPWidenRecipe;
@@ -47,7 +48,7 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
- VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
+ VPTypeAnalysis(VPlan &Plan, LLVMContext &Ctx);
/// Infer the type of \p V. Returns the scalar type of \p V.
Type *inferScalarType(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 71f5285f90236b..4f4febcd1403f8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -502,7 +502,7 @@ static VPValue *createScalarIVSteps(VPlan &Plan, const InductionDescriptor &ID,
}
// Truncate base induction if needed.
- VPTypeAnalysis TypeInfo(SE.getContext());
+ VPTypeAnalysis TypeInfo(Plan, SE.getContext());
Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
if (TruncI) {
Type *TruncTy = TruncI->getType();
@@ -880,7 +880,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
#ifndef NDEBUG
// Verify that the cached type info is for both A and its users is still
// accurate by comparing it to freshly computed types.
- VPTypeAnalysis TypeInfo2(TypeInfo.getContext());
+ VPTypeAnalysis TypeInfo2(*R.getParent()->getPlan(), TypeInfo.getContext());
assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
for (VPUser *U : A->users()) {
auto *R = dyn_cast<VPRecipeBase>(U);
@@ -901,7 +901,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
static void simplifyRecipes(VPlan &Plan, LLVMContext &Ctx) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
- VPTypeAnalysis TypeInfo(Ctx);
+ VPTypeAnalysis TypeInfo(Plan, Ctx);
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
simplifyRecipe(R, TypeInfo);
@@ -922,7 +922,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
// other uses have different types for their operands, making them invalidly
// typed.
DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
- VPTypeAnalysis TypeInfo(Ctx);
+ VPTypeAnalysis TypeInfo(Plan, Ctx);
VPBasicBlock *PH = Plan.getEntry();
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
>From c49ca00df7cc085a981a4e0ff6825a3b9c33a928 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 21 Feb 2024 10:05:55 +0000
Subject: [PATCH 2/2] !fixup store canonica iv type in VPTypeAnalysis.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 ++++++++
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +---
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 17 +++++------------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 4 +++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 +++++++----
5 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index e55db2df82b47b..56310dc11786ca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -212,6 +212,14 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() {
return It;
}
+VPTransformState::VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI,
+ DominatorTree *DT, IRBuilderBase &Builder,
+ InnerLoopVectorizer *ILV, VPlan *Plan,
+ LLVMContext &Ctx)
+ : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan),
+ LVer(nullptr),
+ TypeAnalysis(Plan->getCanonicalIV()->getScalarType(), Ctx) {}
+
Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
if (Def->isLiveIn())
return Def->getLiveInIRValue();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ea4d5b7a3991f6..10ee75397c96cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -236,9 +236,7 @@ struct VPIteration {
struct VPTransformState {
VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI,
DominatorTree *DT, IRBuilderBase &Builder,
- InnerLoopVectorizer *ILV, VPlan *Plan, LLVMContext &Ctx)
- : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan),
- LVer(nullptr), TypeAnalysis(*Plan, Ctx) {}
+ InnerLoopVectorizer *ILV, VPlan *Plan, LLVMContext &Ctx);
/// The chosen Vectorization and Unroll Factors of the loop being vectorized.
ElementCount VF;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b71bac15463328..4aeddf47e03ac4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -198,22 +198,15 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
llvm_unreachable("Unhandled opcode");
}
-VPTypeAnalysis::VPTypeAnalysis(VPlan &Plan, LLVMContext &Ctx) : Ctx(Ctx) {
- auto *CanIV = Plan.getCanonicalIV();
- Type *CanIVTy = inferScalarType(CanIV);
- CachedTypes[&Plan.getVectorTripCount()] = CanIVTy;
- CachedTypes[&Plan.getVFxUF()] = CanIVTy;
- CachedTypes[Plan.getTripCount()] = CanIVTy;
- if (auto *BTC = Plan.getBackedgeTakenCount())
- CachedTypes[BTC] = CanIVTy;
-}
-
Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (V->isLiveIn())
- return V->getLiveInIRValue()->getType();
+ if (V->isLiveIn()) {
+ if (auto *IRValue = V->getLiveInIRValue())
+ return IRValue->getType();
+ return CanonicalIVTy;
+ }
Type *ResultTy =
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 6b218a39f605c1..b7940b36166f22 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -36,6 +36,7 @@ class Type;
/// of the previously inferred types.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
+ Type *CanonicalIVTy;
LLVMContext &Ctx;
Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
@@ -48,7 +49,8 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
- VPTypeAnalysis(VPlan &Plan, LLVMContext &Ctx);
+ VPTypeAnalysis(Type *CanonicalIVTy, LLVMContext &Ctx)
+ : CanonicalIVTy(CanonicalIVTy), Ctx(Ctx) {}
/// Infer the type of \p V. Returns the scalar type of \p V.
Type *inferScalarType(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 60a5c13eacaac1..9c3f35112b592f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -513,7 +513,8 @@ static VPValue *createScalarIVSteps(VPlan &Plan, const InductionDescriptor &ID,
}
// Truncate base induction if needed.
- VPTypeAnalysis TypeInfo(Plan, SE.getContext());
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType(),
+ SE.getContext());
Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
if (TruncI) {
Type *TruncTy = TruncI->getType();
@@ -897,7 +898,9 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
#ifndef NDEBUG
// Verify that the cached type info is for both A and its users is still
// accurate by comparing it to freshly computed types.
- VPTypeAnalysis TypeInfo2(*R.getParent()->getPlan(), TypeInfo.getContext());
+ VPTypeAnalysis TypeInfo2(
+ R.getParent()->getPlan()->getCanonicalIV()->getScalarType(),
+ TypeInfo.getContext());
assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
for (VPUser *U : A->users()) {
auto *R = dyn_cast<VPRecipeBase>(U);
@@ -918,7 +921,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
static void simplifyRecipes(VPlan &Plan, LLVMContext &Ctx) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
- VPTypeAnalysis TypeInfo(Plan, Ctx);
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType(), Ctx);
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
simplifyRecipe(R, TypeInfo);
@@ -939,7 +942,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
// other uses have different types for their operands, making them invalidly
// typed.
DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
- VPTypeAnalysis TypeInfo(Plan, Ctx);
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType(), Ctx);
VPBasicBlock *PH = Plan.getEntry();
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
More information about the llvm-commits
mailing list