[llvm] [VPlan] Add initial anlysis to infer scalar type of VPValues. (PR #69013)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 25 05:16:00 PDT 2023
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/69013
>From 22b23c7de44939e470a47be8af077a7c4516b3c5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 10 Oct 2023 20:57:17 +0100
Subject: [PATCH 1/7] [VPlan] Add initial anlysis to infer scalar type of
VPValues.
This patch adds initial type inferrence for VPValues. It infers the
scalar type of a VPValue, by bottom-up traversing through defining recipes until
root nodes with known types are reached (e.g. live-ins or memory
recipes). The types are then propagated top down through operations.
This is intended as building block for a VPlan-based cost model, which
will need access to type information for VPValues/recipes.
Initial testing is done by asserting the inferred type matches the type
of the result value generated for a widen recipe.
---
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 225 ++++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 56 +++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +
5 files changed, 299 insertions(+), 3 deletions(-)
create mode 100644 llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
create mode 100644 llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 998dfd956575d3c..9674094024b9ec7 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMVectorize
Vectorize.cpp
VectorCombine.cpp
VPlan.cpp
+ VPlanAnalysis.cpp
VPlanHCFGBuilder.cpp
VPlanRecipes.cpp
VPlanSLP.cpp
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e65a7ab2cd028ee..ea1f8a5b9d1e9ab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1167,6 +1167,8 @@ class VPWidenRecipe : public VPRecipeWithIRFlags, public VPValue {
/// Produce widened copies of all Ingredients.
void execute(VPTransformState &State) override;
+ unsigned getOpcode() const { return Opcode; }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
@@ -1458,7 +1460,7 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe {
bool isCanonical() const;
/// Returns the scalar type of the induction.
- const Type *getScalarType() const {
+ Type *getScalarType() const {
return Trunc ? Trunc->getType() : IV->getType();
}
};
@@ -2080,7 +2082,7 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
#endif
/// Returns the scalar type of the induction.
- const Type *getScalarType() const {
+ Type *getScalarType() const {
return getOperand(0)->getLiveInIRValue()->getType();
}
@@ -2149,7 +2151,7 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue {
#endif
/// Returns the scalar type of the induction.
- const Type *getScalarType() const {
+ Type *getScalarType() const {
return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDefiningRecipe())
->getScalarType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
new file mode 100644
index 000000000000000..088da81f950425c
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -0,0 +1,225 @@
+//===- VPlanAnalysis.cpp - Various Analyses working on VPlan ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "VPlanAnalysis.h"
+#include "VPlan.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "vplan"
+
+Type *VPTypeAnalysis::inferType(const VPBlendRecipe *R) {
+ return inferType(R->getIncomingValue(0));
+}
+
+Type *VPTypeAnalysis::inferType(const VPInstruction *R) {
+ switch (R->getOpcode()) {
+ case Instruction::Select:
+ return inferType(R->getOperand(1));
+ case VPInstruction::FirstOrderRecurrenceSplice:
+ return inferType(R->getOperand(0));
+ default:
+ llvm_unreachable("Unhandled instruction!");
+ }
+}
+
+Type *VPTypeAnalysis::inferType(const VPInterleaveRecipe *R) { return nullptr; }
+
+Type *VPTypeAnalysis::inferType(const VPReductionPHIRecipe *R) {
+ return R->getOperand(0)->getLiveInIRValue()->getType();
+}
+
+Type *VPTypeAnalysis::inferType(const VPWidenRecipe *R) {
+ unsigned Opcode = R->getOpcode();
+ switch (Opcode) {
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ return IntegerType::get(Ctx, 1);
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::SRem:
+ case Instruction::URem:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::FNeg:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor: {
+ Type *ResTy = inferType(R->getOperand(0));
+ if (Opcode != Instruction::FNeg) {
+ assert(ResTy == inferType(R->getOperand(1)));
+ CachedTypes[R->getOperand(1)] = ResTy;
+ }
+ return ResTy;
+ }
+ case Instruction::Freeze:
+ return inferType(R->getOperand(0));
+ default:
+ // This instruction is not vectorized by simple widening.
+ // LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
+ llvm_unreachable("Unhandled instruction!");
+ }
+
+ return nullptr;
+}
+
+Type *VPTypeAnalysis::inferType(const VPWidenCallRecipe *R) {
+ auto &CI = *cast<CallInst>(R->getUnderlyingInstr());
+ return CI.getType();
+}
+
+Type *VPTypeAnalysis::inferType(const VPWidenIntOrFpInductionRecipe *R) {
+ return R->getScalarType();
+}
+
+Type *VPTypeAnalysis::inferType(const VPWidenMemoryInstructionRecipe *R) {
+ if (R->isStore())
+ return cast<StoreInst>(&R->getIngredient())->getValueOperand()->getType();
+
+ return cast<LoadInst>(&R->getIngredient())->getType();
+}
+
+Type *VPTypeAnalysis::inferType(const VPWidenSelectRecipe *R) {
+ return inferType(R->getOperand(1));
+}
+
+Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
+ switch (R->getUnderlyingInstr()->getOpcode()) {
+ case Instruction::Call: {
+ unsigned CallIdx = R->getNumOperands() - (R->isPredicated() ? 2 : 1);
+ return cast<Function>(R->getOperand(CallIdx)->getLiveInIRValue())
+ ->getReturnType();
+ }
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::SRem:
+ case Instruction::URem:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::FNeg:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ case Instruction::ICmp:
+ case Instruction::FCmp: {
+ Type *ResTy = inferType(R->getOperand(0));
+ assert(ResTy == inferType(R->getOperand(1)));
+ CachedTypes[R->getOperand(1)] = ResTy;
+ return ResTy;
+ }
+ case Instruction::Trunc:
+ case Instruction::SExt:
+ case Instruction::ZExt:
+ case Instruction::FPExt:
+ case Instruction::FPTrunc:
+ return R->getUnderlyingInstr()->getType();
+ case Instruction::ExtractValue: {
+ return R->getUnderlyingValue()->getType();
+ }
+ case Instruction::Freeze:
+ return inferType(R->getOperand(0));
+ case Instruction::Load:
+ return cast<LoadInst>(R->getUnderlyingInstr())->getType();
+ case Instruction::Store:
+ return cast<StoreInst>(R->getUnderlyingInstr())
+ ->getValueOperand()
+ ->getType();
+ default:
+ llvm_unreachable("Unhandled instruction");
+ }
+
+ return nullptr;
+}
+
+Type *VPTypeAnalysis::inferType(const VPValue *V) {
+ auto Iter = CachedTypes.find(V);
+ if (Iter != CachedTypes.end())
+ return Iter->second;
+
+ Type *ResultTy = nullptr;
+ if (V->isLiveIn())
+ ResultTy = V->getLiveInIRValue()->getType();
+ else {
+ const VPRecipeBase *Def = V->getDefiningRecipe();
+ switch (Def->getVPDefID()) {
+ case VPDef::VPBlendSC:
+ ResultTy = inferType(cast<VPBlendRecipe>(Def));
+ break;
+ case VPDef::VPCanonicalIVPHISC:
+ ResultTy = cast<VPCanonicalIVPHIRecipe>(Def)->getScalarType();
+ break;
+ case VPDef::VPFirstOrderRecurrencePHISC:
+ ResultTy = Def->getOperand(0)->getLiveInIRValue()->getType();
+ break;
+ case VPDef::VPInstructionSC:
+ ResultTy = inferType(cast<VPInstruction>(Def));
+ break;
+ case VPDef::VPInterleaveSC:
+ ResultTy = V->getUnderlyingValue()
+ ->getType(); // inferType(cast<VPInterleaveRecipe>(Def));
+ break;
+ case VPDef::VPPredInstPHISC:
+ ResultTy = inferType(Def->getOperand(0));
+ break;
+ case VPDef::VPReductionPHISC:
+ ResultTy = inferType(cast<VPReductionPHIRecipe>(Def));
+ break;
+ case VPDef::VPReplicateSC:
+ ResultTy = inferType(cast<VPReplicateRecipe>(Def));
+ break;
+ case VPDef::VPScalarIVStepsSC:
+ return inferType(Def->getOperand(0));
+ break;
+ case VPDef::VPWidenSC:
+ ResultTy = inferType(cast<VPWidenRecipe>(Def));
+ break;
+ case VPDef::VPWidenPHISC:
+ return inferType(Def->getOperand(0));
+ case VPDef::VPWidenPointerInductionSC:
+ return inferType(Def->getOperand(0));
+ case VPDef::VPWidenCallSC:
+ ResultTy = inferType(cast<VPWidenCallRecipe>(Def));
+ break;
+ case VPDef::VPWidenCastSC:
+ ResultTy = cast<VPWidenCastRecipe>(Def)->getResultType();
+ break;
+ case VPDef::VPWidenGEPSC:
+ ResultTy = PointerType::get(Ctx, 0);
+ break;
+ case VPDef::VPWidenIntOrFpInductionSC:
+ ResultTy = inferType(cast<VPWidenIntOrFpInductionRecipe>(Def));
+ break;
+ case VPDef::VPWidenMemoryInstructionSC:
+ ResultTy = inferType(cast<VPWidenMemoryInstructionRecipe>(Def));
+ break;
+ case VPDef::VPWidenSelectSC:
+ ResultTy = inferType(cast<VPWidenSelectRecipe>(Def));
+ break;
+ }
+ }
+ CachedTypes[V] = ResultTy;
+ return ResultTy;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
new file mode 100644
index 000000000000000..8fcbe9ca99bb4d5
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -0,0 +1,56 @@
+//===- VPlanAnalysis.h - Various Analyses working on VPlan ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
+
+#include "llvm/ADT/DenseMap.h"
+
+namespace llvm {
+
+class LLVMContext;
+class VPValue;
+class VPBlendRecipe;
+class VPInterleaveRecipe;
+class VPInstruction;
+class VPReductionPHIRecipe;
+class VPWidenRecipe;
+class VPWidenCallRecipe;
+class VPWidenCastRecipe;
+class VPWidenIntOrFpInductionRecipe;
+class VPWidenMemoryInstructionRecipe;
+struct VPWidenSelectRecipe;
+class VPReplicateRecipe;
+class Type;
+
+/// An analysis for type-inferrence for VPValues.
+class VPTypeAnalysis {
+ DenseMap<const VPValue *, Type *> CachedTypes;
+ LLVMContext &Ctx;
+
+ Type *inferType(const VPBlendRecipe *R);
+ Type *inferType(const VPInstruction *R);
+ Type *inferType(const VPInterleaveRecipe *R);
+ Type *inferType(const VPWidenCallRecipe *R);
+ Type *inferType(const VPReductionPHIRecipe *R);
+ Type *inferType(const VPWidenRecipe *R);
+ Type *inferType(const VPWidenIntOrFpInductionRecipe *R);
+ Type *inferType(const VPWidenMemoryInstructionRecipe *R);
+ Type *inferType(const VPWidenSelectRecipe *R);
+ Type *inferType(const VPReplicateRecipe *R);
+
+public:
+ VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
+
+ /// Infer the type of \p V. Returns the scalar type of \p V.
+ Type *inferType(const VPValue *V);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2a1213a98095907..b616abddb00f99a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "VPlan.h"
+#include "VPlanAnalysis.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
@@ -738,7 +739,18 @@ void VPWidenRecipe::execute(VPTransformState &State) {
<< Instruction::getOpcodeName(Opcode));
llvm_unreachable("Unhandled instruction!");
} // end of switch.
+
+#if !defined(NDEBUG)
+ // Verify that VPlan type infererrence results agree with the type of the
+ // generated values.
+ VPTypeAnalysis A(State.Builder.GetInsertBlock()->getContext());
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ assert(VectorType::get(A.inferType(getVPSingleValue()), State.VF) ==
+ State.get(this, Part)->getType());
+ }
+#endif
}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
>From 30a8968d487376efc90c3ec4cfae91d055ab3928 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 20 Oct 2023 12:03:59 +0100
Subject: [PATCH 2/7] Remove Store case for VPReplicateRecipe, as it is dead
code.
---
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 088da81f950425c..8c27787c425cf75 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -143,10 +143,6 @@ Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
return inferType(R->getOperand(0));
case Instruction::Load:
return cast<LoadInst>(R->getUnderlyingInstr())->getType();
- case Instruction::Store:
- return cast<StoreInst>(R->getUnderlyingInstr())
- ->getValueOperand()
- ->getType();
default:
llvm_unreachable("Unhandled instruction");
}
>From 58b145e92d8c018f86ed54cae5b5211e71c89441 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 23 Oct 2023 17:19:37 +0100
Subject: [PATCH 3/7] Update llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
Co-authored-by: ayalz <47719489+ayalz at users.noreply.github.com>
---
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 8fcbe9ca99bb4d5..277cf271d51d041 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -28,7 +28,7 @@ struct VPWidenSelectRecipe;
class VPReplicateRecipe;
class Type;
-/// An analysis for type-inferrence for VPValues.
+/// An analysis for type-inference for VPValues.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
LLVMContext &Ctx;
>From d89b7d890a4724b940dde7536e75b4d4beef0475 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 23 Oct 2023 17:20:12 +0100
Subject: [PATCH 4/7] Update llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Co-authored-by: ayalz <47719489+ayalz at users.noreply.github.com>
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b616abddb00f99a..178e1571960a840 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -741,7 +741,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
} // end of switch.
#if !defined(NDEBUG)
- // Verify that VPlan type infererrence results agree with the type of the
+ // Verify that VPlan type inference results agree with the type of the
// generated values.
VPTypeAnalysis A(State.Builder.GetInsertBlock()->getContext());
for (unsigned Part = 0; Part < State.UF; ++Part) {
>From 4ed3b6dfe165aec75f4da8735d392c7aa7ee0822 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 23 Oct 2023 17:24:43 +0100
Subject: [PATCH 5/7] Address comments
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 174 +++++++++---------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 24 +--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
4 files changed, 105 insertions(+), 99 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ea1f8a5b9d1e9ab..653290a036dd235 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2083,7 +2083,7 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
/// Returns the scalar type of the induction.
Type *getScalarType() const {
- return getOperand(0)->getLiveInIRValue()->getType();
+ return getStartValue()->getLiveInIRValue()->getType();
}
/// Returns true if the recipe only uses the first lane of operand \p Op.
@@ -2151,7 +2151,7 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue {
#endif
/// Returns the scalar type of the induction.
- Type *getScalarType() const {
+ const Type *getScalarType() const {
return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDefiningRecipe())
->getScalarType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 8c27787c425cf75..f2e73ca4f63a7eb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -13,28 +13,35 @@ using namespace llvm;
#define DEBUG_TYPE "vplan"
-Type *VPTypeAnalysis::inferType(const VPBlendRecipe *R) {
- return inferType(R->getIncomingValue(0));
+Type *VPTypeAnalysis::inferScalarType(const VPBlendRecipe *R) {
+ Type *ResTy = inferScalarType(R->getIncomingValue(0));
+ for (unsigned I = 1, E = R->getNumIncomingValues(); I != E; ++I) {
+ VPValue *Inc = R->getIncomingValue(I);
+ assert(inferScalarType(Inc) == ResTy &&
+ "different types inferred for different incoming values");
+ CachedTypes[Inc] = ResTy;
+ }
+ return inferScalarType(R->getIncomingValue(0));
}
-Type *VPTypeAnalysis::inferType(const VPInstruction *R) {
+Type *VPTypeAnalysis::inferScalarType(const VPInstruction *R) {
switch (R->getOpcode()) {
- case Instruction::Select:
- return inferType(R->getOperand(1));
+ case Instruction::Select: {
+ Type *ResTy = inferScalarType(R->getOperand(1));
+ VPValue *OtherV = R->getOperand(2);
+ assert(inferScalarType(OtherV) == ResTy &&
+ "different types inferred for different operands");
+ CachedTypes[OtherV] = ResTy;
+ return ResTy;
+ }
case VPInstruction::FirstOrderRecurrenceSplice:
- return inferType(R->getOperand(0));
+ return inferScalarType(R->getOperand(0));
default:
llvm_unreachable("Unhandled instruction!");
}
}
-Type *VPTypeAnalysis::inferType(const VPInterleaveRecipe *R) { return nullptr; }
-
-Type *VPTypeAnalysis::inferType(const VPReductionPHIRecipe *R) {
- return R->getOperand(0)->getLiveInIRValue()->getType();
-}
-
-Type *VPTypeAnalysis::inferType(const VPWidenRecipe *R) {
+Type *VPTypeAnalysis::inferScalarType(const VPWidenRecipe *R) {
unsigned Opcode = R->getOpcode();
switch (Opcode) {
case Instruction::ICmp:
@@ -48,7 +55,6 @@ Type *VPTypeAnalysis::inferType(const VPWidenRecipe *R) {
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
- case Instruction::FNeg:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::FDiv:
@@ -59,45 +65,47 @@ Type *VPTypeAnalysis::inferType(const VPWidenRecipe *R) {
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
- Type *ResTy = inferType(R->getOperand(0));
- if (Opcode != Instruction::FNeg) {
- assert(ResTy == inferType(R->getOperand(1)));
- CachedTypes[R->getOperand(1)] = ResTy;
- }
+ Type *ResTy = inferScalarType(R->getOperand(0));
+ assert(ResTy == inferScalarType(R->getOperand(1)) &&
+ "types for both operands must match for binary op");
+ CachedTypes[R->getOperand(1)] = ResTy;
return ResTy;
}
+ case Instruction::FNeg:
case Instruction::Freeze:
- return inferType(R->getOperand(0));
+ return inferScalarType(R->getOperand(0));
default:
- // This instruction is not vectorized by simple widening.
- // LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
- llvm_unreachable("Unhandled instruction!");
+ break;
}
- return nullptr;
+ // Type inferrence not implemented for opcode.
+ LLVM_DEBUG(dbgs() << "LV: Found unhandled opcode: "
+ << Instruction::getOpcodeName(Opcode));
+ llvm_unreachable("Unhandled opcode!");
}
-Type *VPTypeAnalysis::inferType(const VPWidenCallRecipe *R) {
+Type *VPTypeAnalysis::inferScalarType(const VPWidenCallRecipe *R) {
auto &CI = *cast<CallInst>(R->getUnderlyingInstr());
return CI.getType();
}
-Type *VPTypeAnalysis::inferType(const VPWidenIntOrFpInductionRecipe *R) {
- return R->getScalarType();
-}
-
-Type *VPTypeAnalysis::inferType(const VPWidenMemoryInstructionRecipe *R) {
+Type *VPTypeAnalysis::inferScalarType(const VPWidenMemoryInstructionRecipe *R) {
if (R->isStore())
return cast<StoreInst>(&R->getIngredient())->getValueOperand()->getType();
return cast<LoadInst>(&R->getIngredient())->getType();
}
-Type *VPTypeAnalysis::inferType(const VPWidenSelectRecipe *R) {
- return inferType(R->getOperand(1));
+Type *VPTypeAnalysis::inferScalarType(const VPWidenSelectRecipe *R) {
+ Type *ResTy = inferScalarType(R->getOperand(1));
+ VPValue *OtherV = R->getOperand(2);
+ assert(inferScalarType(OtherV) == ResTy &&
+ "different types inferred for different operands");
+ CachedTypes[OtherV] = ResTy;
+ return ResTy;
}
-Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
+Type *VPTypeAnalysis::inferScalarType(const VPReplicateRecipe *R) {
switch (R->getUnderlyingInstr()->getOpcode()) {
case Instruction::Call: {
unsigned CallIdx = R->getNumOperands() - (R->isPredicated() ? 2 : 1);
@@ -112,7 +120,6 @@ Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
- case Instruction::FNeg:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::FDiv:
@@ -125,8 +132,8 @@ Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
case Instruction::Xor:
case Instruction::ICmp:
case Instruction::FCmp: {
- Type *ResTy = inferType(R->getOperand(0));
- assert(ResTy == inferType(R->getOperand(1)));
+ Type *ResTy = inferScalarType(R->getOperand(0));
+ assert(ResTy == inferScalarType(R->getOperand(1)));
CachedTypes[R->getOperand(1)] = ResTy;
return ResTy;
}
@@ -137,85 +144,82 @@ Type *VPTypeAnalysis::inferType(const VPReplicateRecipe *R) {
case Instruction::FPTrunc:
return R->getUnderlyingInstr()->getType();
case Instruction::ExtractValue: {
- return R->getUnderlyingValue()->getType();
+ return R->getUnderlyingInstr()->getType();
}
case Instruction::Freeze:
- return inferType(R->getOperand(0));
+ case Instruction::FNeg:
+ return inferScalarType(R->getOperand(0));
case Instruction::Load:
return cast<LoadInst>(R->getUnderlyingInstr())->getType();
default:
- llvm_unreachable("Unhandled instruction");
+ break;
}
- return nullptr;
+ llvm_unreachable("Unhandled instruction");
}
-Type *VPTypeAnalysis::inferType(const VPValue *V) {
- auto Iter = CachedTypes.find(V);
- if (Iter != CachedTypes.end())
- return Iter->second;
+Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
+ if (Type *CachedTy = CachedTypes.lookup(V))
+ return CachedTy;
Type *ResultTy = nullptr;
if (V->isLiveIn())
- ResultTy = V->getLiveInIRValue()->getType();
- else {
+ return V->getLiveInIRValue()->getType();
+
const VPRecipeBase *Def = V->getDefiningRecipe();
switch (Def->getVPDefID()) {
- case VPDef::VPBlendSC:
- ResultTy = inferType(cast<VPBlendRecipe>(Def));
- break;
case VPDef::VPCanonicalIVPHISC:
- ResultTy = cast<VPCanonicalIVPHIRecipe>(Def)->getScalarType();
- break;
case VPDef::VPFirstOrderRecurrencePHISC:
- ResultTy = Def->getOperand(0)->getLiveInIRValue()->getType();
- break;
+ case VPDef::VPReductionPHISC:
+ case VPDef::VPWidenPointerInductionSC:
+ // Handle header phi recipes, except VPWienIntOrFpInduction which needs
+ // special handling due it being possibly truncated.
+ ResultTy = cast<VPHeaderPHIRecipe>(Def)
+ ->getStartValue()
+ ->getLiveInIRValue()
+ ->getType();
+ break;
+ case VPDef::VPWidenIntOrFpInductionSC:
+ ResultTy = cast<VPWidenIntOrFpInductionRecipe>(Def)->getScalarType();
+ break;
+ case VPDef::VPPredInstPHISC:
+ case VPDef::VPScalarIVStepsSC:
+ case VPDef::VPWidenPHISC:
+ ResultTy = inferScalarType(Def->getOperand(0));
+ break;
+ case VPDef::VPBlendSC:
+ ResultTy = inferScalarType(cast<VPBlendRecipe>(Def));
+ break;
case VPDef::VPInstructionSC:
- ResultTy = inferType(cast<VPInstruction>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPInstruction>(Def));
+ break;
case VPDef::VPInterleaveSC:
- ResultTy = V->getUnderlyingValue()
- ->getType(); // inferType(cast<VPInterleaveRecipe>(Def));
- break;
- case VPDef::VPPredInstPHISC:
- ResultTy = inferType(Def->getOperand(0));
- break;
- case VPDef::VPReductionPHISC:
- ResultTy = inferType(cast<VPReductionPHIRecipe>(Def));
- break;
+ // TODO: Use info from interleave group.
+ ResultTy = V->getUnderlyingValue()->getType();
+ break;
case VPDef::VPReplicateSC:
- ResultTy = inferType(cast<VPReplicateRecipe>(Def));
- break;
- case VPDef::VPScalarIVStepsSC:
- return inferType(Def->getOperand(0));
- break;
+ ResultTy = inferScalarType(cast<VPReplicateRecipe>(Def));
+ break;
case VPDef::VPWidenSC:
- ResultTy = inferType(cast<VPWidenRecipe>(Def));
- break;
- case VPDef::VPWidenPHISC:
- return inferType(Def->getOperand(0));
- case VPDef::VPWidenPointerInductionSC:
- return inferType(Def->getOperand(0));
+ ResultTy = inferScalarType(cast<VPWidenRecipe>(Def));
+ break;
case VPDef::VPWidenCallSC:
- ResultTy = inferType(cast<VPWidenCallRecipe>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPWidenCallRecipe>(Def));
+ break;
case VPDef::VPWidenCastSC:
ResultTy = cast<VPWidenCastRecipe>(Def)->getResultType();
break;
case VPDef::VPWidenGEPSC:
ResultTy = PointerType::get(Ctx, 0);
break;
- case VPDef::VPWidenIntOrFpInductionSC:
- ResultTy = inferType(cast<VPWidenIntOrFpInductionRecipe>(Def));
- break;
case VPDef::VPWidenMemoryInstructionSC:
- ResultTy = inferType(cast<VPWidenMemoryInstructionRecipe>(Def));
+ ResultTy = inferScalarType(cast<VPWidenMemoryInstructionRecipe>(Def));
break;
case VPDef::VPWidenSelectSC:
- ResultTy = inferType(cast<VPWidenSelectRecipe>(Def));
+ ResultTy = inferScalarType(cast<VPWidenSelectRecipe>(Def));
break;
}
- }
- CachedTypes[V] = ResultTy;
- return ResultTy;
+ assert(ResultTy && "could not infer type for the given VPValue");
+ CachedTypes[V] = ResultTy;
+ return ResultTy;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 277cf271d51d041..8f1cdf00f5cc0b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -29,26 +29,28 @@ class VPReplicateRecipe;
class Type;
/// An analysis for type-inference for VPValues.
+/// It infers the scalar type for a given VPValue by bottom-up traversing
+/// through defining recipes until root nodes with known types are reached (e.g.
+/// live-ins or memory recipes). The types are then propagated top down through
+/// operations.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
LLVMContext &Ctx;
- Type *inferType(const VPBlendRecipe *R);
- Type *inferType(const VPInstruction *R);
- Type *inferType(const VPInterleaveRecipe *R);
- Type *inferType(const VPWidenCallRecipe *R);
- Type *inferType(const VPReductionPHIRecipe *R);
- Type *inferType(const VPWidenRecipe *R);
- Type *inferType(const VPWidenIntOrFpInductionRecipe *R);
- Type *inferType(const VPWidenMemoryInstructionRecipe *R);
- Type *inferType(const VPWidenSelectRecipe *R);
- Type *inferType(const VPReplicateRecipe *R);
+ Type *inferScalarType(const VPBlendRecipe *R);
+ Type *inferScalarType(const VPInstruction *R);
+ Type *inferScalarType(const VPWidenCallRecipe *R);
+ Type *inferScalarType(const VPWidenRecipe *R);
+ Type *inferScalarType(const VPWidenIntOrFpInductionRecipe *R);
+ Type *inferScalarType(const VPWidenMemoryInstructionRecipe *R);
+ Type *inferScalarType(const VPWidenSelectRecipe *R);
+ Type *inferScalarType(const VPReplicateRecipe *R);
public:
VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
/// Infer the type of \p V. Returns the scalar type of \p V.
- Type *inferType(const VPValue *V);
+ Type *inferScalarType(const VPValue *V);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 178e1571960a840..12ab2e3e0df7020 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -745,7 +745,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
// generated values.
VPTypeAnalysis A(State.Builder.GetInsertBlock()->getContext());
for (unsigned Part = 0; Part < State.UF; ++Part) {
- assert(VectorType::get(A.inferType(getVPSingleValue()), State.VF) ==
+ assert(VectorType::get(A.inferScalarType(getVPSingleValue()), State.VF) ==
State.get(this, Part)->getType());
}
#endif
>From 529805c13b2dbd62fc860174c2a231b0c7a93f8b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 23 Oct 2023 19:42:35 +0100
Subject: [PATCH 6/7] Fix formatting.
---
.../Transforms/Vectorize/VPlanAnalysis.cpp | 48 +++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index f2e73ca4f63a7eb..1b73b270d0aa2cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -172,40 +172,40 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
case VPDef::VPFirstOrderRecurrencePHISC:
case VPDef::VPReductionPHISC:
case VPDef::VPWidenPointerInductionSC:
- // Handle header phi recipes, except VPWienIntOrFpInduction which needs
- // special handling due it being possibly truncated.
- ResultTy = cast<VPHeaderPHIRecipe>(Def)
- ->getStartValue()
- ->getLiveInIRValue()
- ->getType();
- break;
+ // Handle header phi recipes, except VPWienIntOrFpInduction which needs
+ // special handling due it being possibly truncated.
+ ResultTy = cast<VPHeaderPHIRecipe>(Def)
+ ->getStartValue()
+ ->getLiveInIRValue()
+ ->getType();
+ break;
case VPDef::VPWidenIntOrFpInductionSC:
- ResultTy = cast<VPWidenIntOrFpInductionRecipe>(Def)->getScalarType();
- break;
+ ResultTy = cast<VPWidenIntOrFpInductionRecipe>(Def)->getScalarType();
+ break;
case VPDef::VPPredInstPHISC:
case VPDef::VPScalarIVStepsSC:
case VPDef::VPWidenPHISC:
- ResultTy = inferScalarType(Def->getOperand(0));
- break;
+ ResultTy = inferScalarType(Def->getOperand(0));
+ break;
case VPDef::VPBlendSC:
- ResultTy = inferScalarType(cast<VPBlendRecipe>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPBlendRecipe>(Def));
+ break;
case VPDef::VPInstructionSC:
- ResultTy = inferScalarType(cast<VPInstruction>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPInstruction>(Def));
+ break;
case VPDef::VPInterleaveSC:
- // TODO: Use info from interleave group.
- ResultTy = V->getUnderlyingValue()->getType();
- break;
+ // TODO: Use info from interleave group.
+ ResultTy = V->getUnderlyingValue()->getType();
+ break;
case VPDef::VPReplicateSC:
- ResultTy = inferScalarType(cast<VPReplicateRecipe>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPReplicateRecipe>(Def));
+ break;
case VPDef::VPWidenSC:
- ResultTy = inferScalarType(cast<VPWidenRecipe>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPWidenRecipe>(Def));
+ break;
case VPDef::VPWidenCallSC:
- ResultTy = inferScalarType(cast<VPWidenCallRecipe>(Def));
- break;
+ ResultTy = inferScalarType(cast<VPWidenCallRecipe>(Def));
+ break;
case VPDef::VPWidenCastSC:
ResultTy = cast<VPWidenCastRecipe>(Def)->getResultType();
break;
>From a75c5624e7a6c60d2ca79023a9984889342b73bd Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 25 Oct 2023 13:04:40 +0100
Subject: [PATCH 7/7] Address comments
---
.../Transforms/Vectorize/VPlanAnalysis.cpp | 125 +++++++++---------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 3 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +-
3 files changed, 68 insertions(+), 63 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 1b73b270d0aa2cb..5d325fb70e1b3b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -21,7 +21,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPBlendRecipe *R) {
"different types inferred for different incoming values");
CachedTypes[Inc] = ResTy;
}
- return inferScalarType(R->getIncomingValue(0));
+ return ResTy;
}
Type *VPTypeAnalysis::inferScalarType(const VPInstruction *R) {
@@ -37,8 +37,9 @@ Type *VPTypeAnalysis::inferScalarType(const VPInstruction *R) {
case VPInstruction::FirstOrderRecurrenceSplice:
return inferScalarType(R->getOperand(0));
default:
- llvm_unreachable("Unhandled instruction!");
+ break;
}
+ llvm_unreachable("Unhandled opcode!");
}
Type *VPTypeAnalysis::inferScalarType(const VPWidenRecipe *R) {
@@ -133,7 +134,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPReplicateRecipe *R) {
case Instruction::ICmp:
case Instruction::FCmp: {
Type *ResTy = inferScalarType(R->getOperand(0));
- assert(ResTy == inferScalarType(R->getOperand(1)));
+ assert(ResTy == inferScalarType(R->getOperand(1)) &&
+ "inferred types for operands of binary op don't match");
CachedTypes[R->getOperand(1)] = ResTy;
return ResTy;
}
@@ -142,10 +144,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPReplicateRecipe *R) {
case Instruction::ZExt:
case Instruction::FPExt:
case Instruction::FPTrunc:
+ case Instruction::ExtractValue:
return R->getUnderlyingInstr()->getType();
- case Instruction::ExtractValue: {
- return R->getUnderlyingInstr()->getType();
- }
case Instruction::Freeze:
case Instruction::FNeg:
return inferScalarType(R->getOperand(0));
@@ -166,60 +166,61 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (V->isLiveIn())
return V->getLiveInIRValue()->getType();
- const VPRecipeBase *Def = V->getDefiningRecipe();
- switch (Def->getVPDefID()) {
- case VPDef::VPCanonicalIVPHISC:
- case VPDef::VPFirstOrderRecurrencePHISC:
- case VPDef::VPReductionPHISC:
- case VPDef::VPWidenPointerInductionSC:
- // Handle header phi recipes, except VPWienIntOrFpInduction which needs
- // special handling due it being possibly truncated.
- ResultTy = cast<VPHeaderPHIRecipe>(Def)
- ->getStartValue()
- ->getLiveInIRValue()
- ->getType();
- break;
- case VPDef::VPWidenIntOrFpInductionSC:
- ResultTy = cast<VPWidenIntOrFpInductionRecipe>(Def)->getScalarType();
- break;
- case VPDef::VPPredInstPHISC:
- case VPDef::VPScalarIVStepsSC:
- case VPDef::VPWidenPHISC:
- ResultTy = inferScalarType(Def->getOperand(0));
- break;
- case VPDef::VPBlendSC:
- ResultTy = inferScalarType(cast<VPBlendRecipe>(Def));
- break;
- case VPDef::VPInstructionSC:
- ResultTy = inferScalarType(cast<VPInstruction>(Def));
- break;
- case VPDef::VPInterleaveSC:
- // TODO: Use info from interleave group.
- ResultTy = V->getUnderlyingValue()->getType();
- break;
- case VPDef::VPReplicateSC:
- ResultTy = inferScalarType(cast<VPReplicateRecipe>(Def));
- break;
- case VPDef::VPWidenSC:
- ResultTy = inferScalarType(cast<VPWidenRecipe>(Def));
- break;
- case VPDef::VPWidenCallSC:
- ResultTy = inferScalarType(cast<VPWidenCallRecipe>(Def));
- break;
- case VPDef::VPWidenCastSC:
- ResultTy = cast<VPWidenCastRecipe>(Def)->getResultType();
- break;
- case VPDef::VPWidenGEPSC:
- ResultTy = PointerType::get(Ctx, 0);
- break;
- case VPDef::VPWidenMemoryInstructionSC:
- ResultTy = inferScalarType(cast<VPWidenMemoryInstructionRecipe>(Def));
- break;
- case VPDef::VPWidenSelectSC:
- ResultTy = inferScalarType(cast<VPWidenSelectRecipe>(Def));
- break;
- }
- assert(ResultTy && "could not infer type for the given VPValue");
- CachedTypes[V] = ResultTy;
- return ResultTy;
+ const VPRecipeBase *Def = V->getDefiningRecipe();
+
+ switch (Def->getVPDefID()) {
+ case VPDef::VPCanonicalIVPHISC:
+ case VPDef::VPFirstOrderRecurrencePHISC:
+ case VPDef::VPReductionPHISC:
+ case VPDef::VPWidenPointerInductionSC:
+ // Handle header phi recipes, except VPWienIntOrFpInduction which needs
+ // special handling due it being possibly truncated.
+ ResultTy = cast<VPHeaderPHIRecipe>(Def)
+ ->getStartValue()
+ ->getLiveInIRValue()
+ ->getType();
+ break;
+ case VPDef::VPWidenIntOrFpInductionSC:
+ ResultTy = cast<VPWidenIntOrFpInductionRecipe>(Def)->getScalarType();
+ break;
+ case VPDef::VPPredInstPHISC:
+ case VPDef::VPScalarIVStepsSC:
+ case VPDef::VPWidenPHISC:
+ ResultTy = inferScalarType(Def->getOperand(0));
+ break;
+ case VPDef::VPBlendSC:
+ ResultTy = inferScalarType(cast<VPBlendRecipe>(Def));
+ break;
+ case VPDef::VPInstructionSC:
+ ResultTy = inferScalarType(cast<VPInstruction>(Def));
+ break;
+ case VPDef::VPInterleaveSC:
+ // TODO: Use info from interleave group.
+ ResultTy = V->getUnderlyingValue()->getType();
+ break;
+ case VPDef::VPReplicateSC:
+ ResultTy = inferScalarType(cast<VPReplicateRecipe>(Def));
+ break;
+ case VPDef::VPWidenSC:
+ ResultTy = inferScalarType(cast<VPWidenRecipe>(Def));
+ break;
+ case VPDef::VPWidenCallSC:
+ ResultTy = inferScalarType(cast<VPWidenCallRecipe>(Def));
+ break;
+ case VPDef::VPWidenCastSC:
+ ResultTy = cast<VPWidenCastRecipe>(Def)->getResultType();
+ break;
+ case VPDef::VPWidenGEPSC:
+ ResultTy = PointerType::get(Ctx, 0);
+ break;
+ case VPDef::VPWidenMemoryInstructionSC:
+ ResultTy = inferScalarType(cast<VPWidenMemoryInstructionRecipe>(Def));
+ break;
+ case VPDef::VPWidenSelectSC:
+ ResultTy = inferScalarType(cast<VPWidenSelectRecipe>(Def));
+ break;
+ }
+ assert(ResultTy && "could not infer type for the given VPValue");
+ CachedTypes[V] = ResultTy;
+ return ResultTy;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 8f1cdf00f5cc0b6..7c223084669596b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -33,6 +33,9 @@ class Type;
/// through defining recipes until root nodes with known types are reached (e.g.
/// live-ins or memory recipes). The types are then propagated top down through
/// operations.
+/// Note that the analysis caches the infered types. A new analysis object must
+/// be constructed once a VPlan has been modified in a way that invalidates any
+/// of the previously infered types.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
LLVMContext &Ctx;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 12ab2e3e0df7020..3b44aa993e9b480 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -746,7 +746,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
VPTypeAnalysis A(State.Builder.GetInsertBlock()->getContext());
for (unsigned Part = 0; Part < State.UF; ++Part) {
assert(VectorType::get(A.inferScalarType(getVPSingleValue()), State.VF) ==
- State.get(this, Part)->getType());
+ State.get(this, Part)->getType() &&
+ "infered type and type from generated instructions do not match");
}
#endif
}
More information about the llvm-commits
mailing list