fhahn created this revision.
fhahn added reviewers: gilr, rengolin, Ayal.
Herald added subscribers: tschuett, psnobl, rogfer01, bollu, hiraditya.
Herald added a project: All.
fhahn requested review of this revision.
Herald added subscribers: pcwang-thead, vkmr.
Herald added a project: LLVM.
Update sinkScalarOperands to consider all operands of recipes in
replicate blocks as sink candidates This enables additional sinking
opportunities and is another step towards retiring LLVM IR-based
sinkScalarOperands.
Depends on D139788 <https://reviews.llvm.org/D139788>.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D139790
Files:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
Index: llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -22,7 +22,6 @@
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK-NEXT: Successor(s): loop.0
@@ -35,6 +34,7 @@
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>
@@ -1076,7 +1076,6 @@
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]>
; CHECK-NEXT: Successor(s): loop.0
@@ -1090,6 +1089,7 @@
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr ir<%src>, vp<[[SCALAR_STEPS]]>
; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>
; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
Index: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -109,17 +109,18 @@
auto Iter = depth_first(
VPBlockRecursiveTraversalWrapper<VPBlockBase *>(Plan.getEntry()));
bool Changed = false;
- // First, collect the operands of all predicated replicate recipes as seeds
- // for sinking.
+ // First, collect the operands of all recipes in replicate blocks as seeds for
+ // sinking.
SetVector<std::pair<VPBasicBlock *, VPRecipeBase *>> WorkList;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
+ VPRegionBlock *Parent = VPBB->getParent();
+ if (!Parent || !Parent->isReplicator() ||
+ VPBB->getPredecessors().size() != 1)
+ continue;
for (auto &Recipe : *VPBB) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
- if (!RepR || !RepR->isPredicated())
- continue;
- for (VPValue *Op : RepR->operands())
+ for (VPValue *Op : Recipe.operands())
if (auto *Def = Op->getDefiningRecipe())
- WorkList.insert(std::make_pair(RepR->getParent(), Def));
+ WorkList.insert(std::make_pair(Recipe.getParent(), Def));
}
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D139790.481920.patch
Type: text/x-patch
Size: 3321 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20221211/f80dae37/attachment.bin>