[llvm] cd608dc - [VPlan] Use VPDef for VPInterleaveRecipe.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 21 03:02:26 PST 2020
Author: Florian Hahn
Date: 2020-12-21T10:56:53Z
New Revision: cd608dc8d3e975fa3c57327d2146b5c223bcf83a
URL: https://github.com/llvm/llvm-project/commit/cd608dc8d3e975fa3c57327d2146b5c223bcf83a
DIFF: https://github.com/llvm/llvm-project/commit/cd608dc8d3e975fa3c57327d2146b5c223bcf83a.diff
LOG: [VPlan] Use VPDef for VPInterleaveRecipe.
This patch turns updates VPInterleaveRecipe to manage the values it defines
using VPDef. The VPValue is used during VPlan construction and
codegeneration instead of the plain IR reference where possible.
Reviewed By: gilr
Differential Revision: https://reviews.llvm.org/D90562
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e486f7110295..25deab6d2b35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -569,6 +569,7 @@ class InnerLoopVectorizer {
/// BlockInMask is non-null. Use \p State to translate given VPValues to IR
/// values in the vectorized loop.
void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
+ ArrayRef<VPValue *> VPDefs,
VPTransformState &State, VPValue *Addr,
ArrayRef<VPValue *> StoredValues,
VPValue *BlockInMask = nullptr);
@@ -2514,8 +2515,9 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
void InnerLoopVectorizer::vectorizeInterleaveGroup(
- const InterleaveGroup<Instruction> *Group, VPTransformState &State,
- VPValue *Addr, ArrayRef<VPValue *> StoredValues, VPValue *BlockInMask) {
+ const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
+ VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
+ VPValue *BlockInMask) {
Instruction *Instr = Group->getInsertPos();
const DataLayout &DL = Instr->getModule()->getDataLayout();
@@ -2617,6 +2619,7 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(
// For each member in the group, shuffle out the appropriate data from the
// wide loads.
+ unsigned J = 0;
for (unsigned I = 0; I < InterleaveFactor; ++I) {
Instruction *Member = Group->getMember(I);
@@ -2641,8 +2644,9 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(
if (Group->isReverse())
StridedVec = reverseVector(StridedVec);
- VectorLoopValueMap.setVectorValue(Member, Part, StridedVec);
+ State.set(VPDefs[J], Member, StridedVec, Part);
}
+ ++J;
}
return;
}
@@ -7980,9 +7984,8 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
return BlockMaskCache[BB] = BlockMask;
}
-VPWidenMemoryInstructionRecipe *
-VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
- VPlanPtr &Plan) {
+VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
+ VPlanPtr &Plan) {
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Must be called with either a load or store");
@@ -8472,16 +8475,17 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i)))
StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0)));
- (new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
- Recipe->getMask()))
- ->insertBefore(Recipe);
-
+ auto *VPIG = new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
+ Recipe->getMask());
+ VPIG->insertBefore(Recipe);
+ unsigned J = 0;
for (unsigned i = 0; i < IG->getFactor(); ++i)
if (Instruction *Member = IG->getMember(i)) {
if (!Member->getType()->isVoidTy()) {
VPValue *OriginalV = Plan->getVPValue(Member);
Plan->removeVPValueFor(Member);
- OriginalV->replaceAllUsesWith(Plan->getOrAddVPValue(Member));
+ OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
+ J++;
}
RecipeBuilder.getRecipe(Member)->eraseFromParent();
}
@@ -8713,8 +8717,8 @@ void VPBlendRecipe::execute(VPTransformState &State) {
void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Interleave group being replicated.");
- State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getStoredValues(),
- getMask());
+ State.ILV->vectorizeInterleaveGroup(IG, definedValues(), State, getAddr(),
+ getStoredValues(), getMask());
}
void VPReductionRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 6f055ca80ff2..41679637167d 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -61,8 +61,8 @@ class VPRecipeBuilder {
/// Check if the load or store instruction \p I should widened for \p
/// Range.Start and potentially masked. Such instructions are handled by a
/// recipe that takes an additional VPInstruction for the mask.
- VPWidenMemoryInstructionRecipe *
- tryToWidenMemory(Instruction *I, VFRange &Range, VPlanPtr &Plan);
+ VPRecipeBase *tryToWidenMemory(Instruction *I, VFRange &Range,
+ VPlanPtr &Plan);
/// Check if an induction recipe should be constructed for \I. If so build and
/// return it. If not, return null.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 516a07998854..e58b49a64737 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -384,8 +384,12 @@ void VPBasicBlock::execute(VPTransformState *State) {
void VPBasicBlock::dropAllReferences(VPValue *NewValue) {
for (VPRecipeBase &R : Recipes) {
- if (auto *VPV = R.toVPValue())
- VPV->replaceAllUsesWith(NewValue);
+ if (VPValue *Def = R.toVPValue())
+ Def->replaceAllUsesWith(NewValue);
+ else if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
+ for (auto *Def : IR->definedValues())
+ Def->replaceAllUsesWith(NewValue);
+ }
if (auto *User = R.toVPUser())
for (unsigned I = 0, E = User->getNumOperands(); I != E; I++)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 02ae60990c63..37f1e9e73c39 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1047,7 +1047,7 @@ class VPBlendRecipe : public VPRecipeBase, public VPUser {
/// or stores into one wide load/store and shuffles. The first operand of a
/// VPInterleave recipe is the address, followed by the stored values, followed
/// by an optional mask.
-class VPInterleaveRecipe : public VPRecipeBase, public VPUser {
+class VPInterleaveRecipe : public VPRecipeBase, public VPDef, public VPUser {
const InterleaveGroup<Instruction> *IG;
bool HasMask = false;
@@ -1055,7 +1055,14 @@ class VPInterleaveRecipe : public VPRecipeBase, public VPUser {
public:
VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
ArrayRef<VPValue *> StoredValues, VPValue *Mask)
- : VPRecipeBase(VPInterleaveSC), VPUser({Addr}), IG(IG) {
+ : VPRecipeBase(VPInterleaveSC), VPUser(Addr), IG(IG) {
+ for (unsigned i = 0; i < IG->getFactor(); ++i)
+ if (Instruction *I = IG->getMember(i)) {
+ if (I->getType()->isVoidTy())
+ continue;
+ new VPValue(I, this);
+ }
+
for (auto *SV : StoredValues)
addOperand(SV);
if (Mask) {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index f89abc8769b4..4864d52ad01e 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "gtest/gtest.h"
@@ -470,7 +471,8 @@ TEST(VPRecipeTest, CastVPInterleaveRecipeToVPUser) {
VPValue Addr;
VPValue Mask;
- VPInterleaveRecipe Recipe(nullptr, &Addr, {}, &Mask);
+ InterleaveGroup<Instruction> IG(4, false, Align(4));
+ VPInterleaveRecipe Recipe(&IG, &Addr, {}, &Mask);
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
More information about the llvm-commits
mailing list