[llvm] 0676156 - Revert "[VPlan] Also consider operands of sink candidates in same block."
Augie Fackler via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 4 13:18:46 PST 2023
Author: Augie Fackler
Date: 2023-01-04T16:17:13-05:00
New Revision: 0676156f8143e2934f9ba0325daf82791e0235a1
URL: https://github.com/llvm/llvm-project/commit/0676156f8143e2934f9ba0325daf82791e0235a1
DIFF: https://github.com/llvm/llvm-project/commit/0676156f8143e2934f9ba0325daf82791e0235a1.diff
LOG: Revert "[VPlan] Also consider operands of sink candidates in same block."
This reverts commit aa2414729ebbcb2d8f162e9002a3a6aa768b1f9d.
Previously-valid IR from a tensorflow test case (as shown on the
Diffusion revision for aa2414729ebbcb2d8f162e9002a3a6aa768b1f9d) started
hanging in the loop-vectorize pass. Reverting to keep everyone working.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c684099b3add5..057a328083a61 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -113,18 +113,14 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
// First, collect the operands of all predicated replicate recipes as seeds
// for sinking.
SetVector<std::pair<VPBasicBlock *, VPRecipeBase *>> WorkList;
- auto QueueOperands = [&WorkList](VPRecipeBase *R) {
- for (VPValue *Op : R->operands())
- if (auto *Def = Op->getDefiningRecipe())
- WorkList.insert(std::make_pair(R->getParent(), Def));
- };
-
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
for (auto &Recipe : *VPBB) {
auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
if (!RepR || !RepR->isPredicated())
continue;
- QueueOperands(RepR);
+ for (VPValue *Op : RepR->operands())
+ if (auto *Def = Op->getDefiningRecipe())
+ WorkList.insert(std::make_pair(VPBB, Def));
}
}
@@ -134,11 +130,8 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
VPBasicBlock *SinkTo;
VPRecipeBase *SinkCandidate;
std::tie(SinkTo, SinkCandidate) = WorkList.pop_back_val();
- if (SinkCandidate->getParent() == SinkTo) {
- QueueOperands(SinkCandidate);
- continue;
- }
- if (SinkCandidate->mayHaveSideEffects() ||
+ if (SinkCandidate->getParent() == SinkTo ||
+ SinkCandidate->mayHaveSideEffects() ||
SinkCandidate->mayReadOrWriteMemory())
continue;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
@@ -192,7 +185,9 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
}
}
SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
- QueueOperands(SinkCandidate);
+ for (VPValue *Op : SinkCandidate->operands())
+ if (auto *Def = Op->getDefiningRecipe())
+ WorkList.insert(std::make_pair(SinkTo, Def));
Changed = true;
}
return Changed;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 625146802efad..2ccb57329dc3b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -352,6 +352,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y>
; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next>
@@ -364,7 +365,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep>
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 86982cc29ac90..3322ef22ad608 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -72,6 +72,7 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<false> + vp<[[CAN_IV]]> * ir<true>
; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<true>
+; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; DBG-NEXT: Successor(s): pred.store
; DBG-EMPTY:
; DBG-NEXT: <xVFxUF> pred.store: {
@@ -80,7 +81,6 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-NEXT: Successor(s): pred.store.if, pred.store.continue
; DBG-EMPTY:
; DBG-NEXT: pred.store.if:
-; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; DBG-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<[[STEPS2]]>
; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src>
; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 3c552f74eeae0..a98035c618142 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -22,6 +22,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
@@ -31,7 +32,6 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>
@@ -992,6 +992,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
@@ -1002,7 +1003,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr ir<%src>, vp<[[SCALAR_STEPS]]>
; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>
; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
More information about the llvm-commits
mailing list