[llvm] [VPlan] Rewrite sinkScalarOperands (PR #151696)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 3 03:56:36 PDT 2025
================
@@ -122,53 +122,64 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
}
static bool sinkScalarOperands(VPlan &Plan) {
- auto Iter = vp_depth_first_deep(Plan.getEntry());
+ bool ScalarVFOnly = Plan.hasScalarVFOnly();
bool Changed = false;
// First, collect the operands of all recipes in replicate blocks as seeds for
// sinking.
SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
- for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
+ for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(
+ vp_depth_first_deep(Plan.getEntry()))) {
VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
continue;
- VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors()[0]);
+ VPBasicBlock *VPBB =
+ dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors().front());
if (!VPBB || VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
continue;
for (auto &Recipe : *VPBB) {
- for (VPValue *Op : Recipe.operands())
- if (auto *Def =
- dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe()))
- WorkList.insert(std::make_pair(VPBB, Def));
+ for (VPValue *Op : Recipe.operands()) {
+ auto *Def =
+ dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe());
+ if (!Def)
+ continue;
+
+ // We only know how to duplicate VPReplicateRecipes and
+ // VPScalarIVStepsRecipes for now.
+ if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Def))
+ continue;
+
+ // TODO: Relax checks in the future, e.g. we could also sink reads if
+ // their memory location is not modified in the vector loop.
+ if (Def->getParent() == VPBB || Def->mayHaveSideEffects() ||
+ Def->mayReadOrWriteMemory())
+ continue;
+
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(Op))
+ if (!ScalarVFOnly && RepR->isSingleScalar())
+ continue;
+
+ WorkList.insert(std::make_pair(VPBB, Def));
+ }
}
}
- bool ScalarVFOnly = Plan.hasScalarVFOnly();
// Try to sink each replicate or scalar IV steps recipe in the worklist.
- for (unsigned I = 0; I != WorkList.size(); ++I) {
+ for (const auto &Item : WorkList) {
VPBasicBlock *SinkTo;
VPSingleDefRecipe *SinkCandidate;
- std::tie(SinkTo, SinkCandidate) = WorkList[I];
- if (SinkCandidate->getParent() == SinkTo ||
- SinkCandidate->mayHaveSideEffects() ||
- SinkCandidate->mayReadOrWriteMemory())
- continue;
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
- if (!ScalarVFOnly && RepR->isSingleScalar())
- continue;
- } else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
- continue;
+ std::tie(SinkTo, SinkCandidate) = Item;
- bool NeedsDuplicating = false;
// All recipe users of the sink candidate must be in the same block SinkTo
// or all users outside of SinkTo must be uniform-after-vectorization (
// i.e., only first lane is used) . In the latter case, we need to duplicate
// SinkCandidate.
- auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
- SinkCandidate](VPUser *U) {
+ bool NeedsDuplicating = false;
+ auto CanSinkWithUser = [SinkTo, SinkCandidate,
+ &NeedsDuplicating](VPUser *U) {
auto *UI = cast<VPRecipeBase>(U);
if (UI->getParent() == SinkTo)
return true;
- NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
+ NeedsDuplicating |= UI->onlyFirstLaneUsed(SinkCandidate);
----------------
artagnon wrote:
Ah, it wasn't a bug after all.
https://github.com/llvm/llvm-project/pull/151696
More information about the llvm-commits
mailing list