[llvm] 105032f - [LV] Use PHI recipe instead of PredRecipe for subsequent uses.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 18 01:35:49 PDT 2022
Author: Florian Hahn
Date: 2022-07-18T09:35:34+01:00
New Revision: 105032f5490817ca678338d0ce964dd06aba178f
URL: https://github.com/llvm/llvm-project/commit/105032f5490817ca678338d0ce964dd06aba178f
DIFF: https://github.com/llvm/llvm-project/commit/105032f5490817ca678338d0ce964dd06aba178f.diff
LOG: [LV] Use PHI recipe instead of PredRecipe for subsequent uses.
At the moment, the VPPRedInstPHIRecipe is not used in subsequent uses of
the predicate recipe. This incorrectly models the def-use chains, as all
later uses should use the phi recipe. Fix that by delaying recording of
the recipe.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D129436
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0e96323d68621..800e9197d0952 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8415,8 +8415,6 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()),
IsUniform, IsPredicated);
- setRecipe(I, Recipe);
- Plan->addVPValue(I, Recipe);
// Find if I uses a predicated instruction. If so, it will use its scalar
// value. Avoid hoisting the insert-element which packs the scalar value into
@@ -8435,6 +8433,8 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
// Finalize the recipe for Instr, first if it is not predicated.
if (!IsPredicated) {
LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
+ setRecipe(I, Recipe);
+ Plan->addVPValue(I, Recipe);
VPBB->appendRecipe(Recipe);
return VPBB;
}
@@ -8445,7 +8445,7 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
"predicated replication.");
VPBlockUtils::disconnectBlocks(VPBB, SingleSucc);
// Record predicated instructions for above packing optimizations.
- VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
+ VPBlockBase *Region = createReplicateRegion(Recipe, Plan);
VPBlockUtils::insertBlockAfter(Region, VPBB);
auto *RegSucc = new VPBasicBlock();
VPBlockUtils::insertBlockAfter(RegSucc, Region);
@@ -8453,11 +8453,12 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
return RegSucc;
}
-VPRegionBlock *VPRecipeBuilder::createReplicateRegion(
- Instruction *Instr, VPReplicateRecipe *PredRecipe, VPlanPtr &Plan) {
+VPRegionBlock *
+VPRecipeBuilder::createReplicateRegion(VPReplicateRecipe *PredRecipe,
+ VPlanPtr &Plan) {
+ Instruction *Instr = PredRecipe->getUnderlyingInstr();
// Instructions marked for predication are replicated and placed under an
// if-then construct to prevent side-effects.
-
// Generate recipes to compute the block mask for this region.
VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan);
@@ -8470,9 +8471,13 @@ VPRegionBlock *VPRecipeBuilder::createReplicateRegion(
? nullptr
: new VPPredInstPHIRecipe(PredRecipe);
if (PHIRecipe) {
- Plan->removeVPValueFor(Instr);
+ setRecipe(Instr, PHIRecipe);
Plan->addVPValue(Instr, PHIRecipe);
+ } else {
+ setRecipe(Instr, PredRecipe);
+ Plan->addVPValue(Instr, PredRecipe);
}
+
auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index c7949c42c03ed..07d3fa56020b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -157,10 +157,8 @@ class VPRecipeBuilder {
return Ingredient2Recipe[I];
}
- /// Create a replicating region for instruction \p I that requires
- /// predication. \p PredRecipe is a VPReplicateRecipe holding \p I.
- VPRegionBlock *createReplicateRegion(Instruction *I,
- VPReplicateRecipe *PredRecipe,
+ /// Create a replicating region for \p PredRecipe.
+ VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
VPlanPtr &Plan);
/// Build a VPReplicationRecipe for \p I and enclose it within a Region if it
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index a2438329e4d4b..55551711bf96f 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -447,7 +447,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, i32* %src) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<%l>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, vp<[[P_L:%.+]]>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]>
; CHECK-NEXT: Successor(s): loop.0
@@ -465,13 +465,13 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, i32* %src) {
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_L:%.+]]> = ir<%l>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_L]]> = ir<%l>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): pred.load.succ
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.succ:
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn> ir<%l>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn> vp<[[P_L]]>
; CHECK-NEXT: Successor(s): pred.sdiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.sdiv: {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index db28fde8e95d0..1b5ca0520ebd0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -851,7 +851,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, ir<%lv.a>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]>
@@ -867,13 +867,13 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> ir<%lv.a>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> vp<[[PRED]]>
; CHECK-NEXT: Successor(s): loop.1
; CHECK-EMPTY:
; CHECK-NEXT: loop.1:
More information about the llvm-commits
mailing list