[llvm] AMDGPU: Extract mergeIncomingLaneMasks from SILowerI1Copies lowerPhis (PR #216995)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 04:31:04 PDT 2026
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/216995
Factor the SSA-updater lane-mask merge core out of lowerPhis into a
helper function for future use. The merge logic is independent of how
the incoming values are collected. NFC
Co-Authored-By: Claude <noreply at anthropic.com> claude-opus-4.8
>From 841d6e4470ddb7985647827af6e0c7c165b9f234 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 18 Aug 2026 12:58:50 +0200
Subject: [PATCH] AMDGPU: Extract mergeIncomingLaneMasks from SILowerI1Copies
lowerPhis
Factor the SSA-updater lane-mask merge core out of lowerPhis into a
helper function for future use. The merge logic is independent of how
the incoming values are collected. NFC
Co-Authored-By: Claude <noreply at anthropic.com> claude-opus-4.8
---
llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp | 169 +++++++++++----------
llvm/lib/Target/AMDGPU/SILowerI1Copies.h | 10 ++
2 files changed, 95 insertions(+), 84 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
index 283d33318b329..28043b1321a2c 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -449,9 +449,92 @@ AMDGPU::PhiLoweringHelper::PhiLoweringHelper(MachineFunction *MF,
TII = ST->getInstrInfo();
}
-bool AMDGPU::PhiLoweringHelper::lowerPhis() {
+void AMDGPU::PhiLoweringHelper::mergeIncomingLaneMasks(
+ Register DstReg, MachineBasicBlock &MBB,
+ SmallVectorImpl<Incoming> &Incomings, MachineIDFSSAUpdater &SSAUpdater) {
LoopFinder LF(*DT, *PDT);
PhiIncomingAnalysis PIA(*PDT, TII);
+ LF.initialize(MBB);
+
+ // Sort the incomings such that incoming values that dominate other incoming
+ // values are sorted earlier. This allows us to do some amount of on-the-fly
+ // constant folding.
+ // Incoming with smaller DFSNumIn goes first, DFSNumIn is 0 for entry block.
+ llvm::sort(Incomings, [this](Incoming LHS, Incoming RHS) {
+ return DT->getNode(LHS.Block)->getDFSNumIn() <
+ DT->getNode(RHS.Block)->getDFSNumIn();
+ });
+
+ // Values in a loop that are observed outside the loop receive a simple but
+ // conservatively correct treatment.
+ std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
+ for (MachineInstr &Use : MRI->use_instructions(DstReg))
+ DomBlocks.push_back(Use.getParent());
+
+ MachineBasicBlock *PostDomBound = PDT->findNearestCommonDominator(DomBlocks);
+
+ // FIXME: This fails to find irreducible cycles. If we have a def (other
+ // than a constant) in a pair of blocks that end up looping back to each
+ // other, it will be mishandle. Due to structurization this shouldn't occur
+ // in practice.
+ unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
+
+ SSAUpdater.addUseBlock(&MBB);
+
+ if (FoundLoopLevel) {
+ LF.addLoopEntries(FoundLoopLevel, SSAUpdater, *MRI, LaneMaskRegAttrs,
+ Incomings);
+
+ for (auto &Incoming : Incomings) {
+ SSAUpdater.addUseBlock(Incoming.Block);
+ Incoming.UpdatedReg = createLaneMaskReg(MRI, LaneMaskRegAttrs);
+ SSAUpdater.addAvailableValue(Incoming.Block, Incoming.UpdatedReg);
+ }
+
+ SSAUpdater.calculate();
+
+ for (auto &Incoming : Incomings) {
+ MachineBasicBlock &IMBB = *Incoming.Block;
+ buildMergeLaneMasks(
+ IMBB, getSaluInsertionAtEnd(IMBB), {}, Incoming.UpdatedReg,
+ SSAUpdater.getValueInMiddleOfBlock(&IMBB), Incoming.Reg);
+ }
+ } else {
+ // The value is not observed from outside a loop. Use a more accurate
+ // lowering.
+ PIA.analyze(MBB, Incomings);
+
+ for (MachineBasicBlock *PredMBB : PIA.predecessors())
+ SSAUpdater.addAvailableValue(
+ PredMBB, insertUndefLaneMask(PredMBB, MRI, LaneMaskRegAttrs));
+
+ for (auto &Incoming : Incomings) {
+ MachineBasicBlock &IMBB = *Incoming.Block;
+ if (PIA.isSource(IMBB)) {
+ constrainAsLaneMask(Incoming);
+ SSAUpdater.addAvailableValue(&IMBB, Incoming.Reg);
+ } else {
+ SSAUpdater.addUseBlock(&IMBB);
+ Incoming.UpdatedReg = createLaneMaskReg(MRI, LaneMaskRegAttrs);
+ SSAUpdater.addAvailableValue(&IMBB, Incoming.UpdatedReg);
+ }
+ }
+
+ SSAUpdater.calculate();
+
+ for (auto &Incoming : Incomings) {
+ if (!Incoming.UpdatedReg.isValid())
+ continue;
+
+ MachineBasicBlock &IMBB = *Incoming.Block;
+ buildMergeLaneMasks(
+ IMBB, getSaluInsertionAtEnd(IMBB), {}, Incoming.UpdatedReg,
+ SSAUpdater.getValueInMiddleOfBlock(&IMBB), Incoming.Reg);
+ }
+ }
+}
+
+bool AMDGPU::PhiLoweringHelper::lowerPhis() {
SmallVector<MachineInstr *, 4> Vreg1Phis;
SmallVector<Incoming, 4> Incomings;
@@ -460,14 +543,8 @@ bool AMDGPU::PhiLoweringHelper::lowerPhis() {
return false;
DT->updateDFSNumbers();
- MachineBasicBlock *PrevMBB = nullptr;
for (MachineInstr *MI : Vreg1Phis) {
MachineBasicBlock &MBB = *MI->getParent();
- if (&MBB != PrevMBB) {
- LF.initialize(MBB);
- PrevMBB = &MBB;
- }
-
LLVM_DEBUG(dbgs() << "Lower PHI: " << *MI);
Register DstReg = MI->getOperand(0).getReg();
@@ -476,88 +553,12 @@ bool AMDGPU::PhiLoweringHelper::lowerPhis() {
collectIncomingValuesFromPhi(MI, Incomings);
- // Sort the incomings such that incoming values that dominate other incoming
- // values are sorted earlier. This allows us to do some amount of on-the-fly
- // constant folding.
- // Incoming with smaller DFSNumIn goes first, DFSNumIn is 0 for entry block.
- llvm::sort(Incomings, [this](Incoming LHS, Incoming RHS) {
- return DT->getNode(LHS.Block)->getDFSNumIn() <
- DT->getNode(RHS.Block)->getDFSNumIn();
- });
-
#ifndef NDEBUG
PhiRegisters.insert(DstReg);
#endif
- // Phis in a loop that are observed outside the loop receive a simple but
- // conservatively correct treatment.
- std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
- for (MachineInstr &Use : MRI->use_instructions(DstReg))
- DomBlocks.push_back(Use.getParent());
-
- MachineBasicBlock *PostDomBound =
- PDT->findNearestCommonDominator(DomBlocks);
-
- // FIXME: This fails to find irreducible cycles. If we have a def (other
- // than a constant) in a pair of blocks that end up looping back to each
- // other, it will be mishandle. Due to structurization this shouldn't occur
- // in practice.
- unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
-
MachineIDFSSAUpdater SSAUpdater(*DT, *MF, DstReg);
- SSAUpdater.addUseBlock(&MBB);
-
- if (FoundLoopLevel) {
- LF.addLoopEntries(FoundLoopLevel, SSAUpdater, *MRI, LaneMaskRegAttrs,
- Incomings);
-
- for (auto &Incoming : Incomings) {
- SSAUpdater.addUseBlock(Incoming.Block);
- Incoming.UpdatedReg = createLaneMaskReg(MRI, LaneMaskRegAttrs);
- SSAUpdater.addAvailableValue(Incoming.Block, Incoming.UpdatedReg);
- }
-
- SSAUpdater.calculate();
-
- for (auto &Incoming : Incomings) {
- MachineBasicBlock &IMBB = *Incoming.Block;
- buildMergeLaneMasks(
- IMBB, getSaluInsertionAtEnd(IMBB), {}, Incoming.UpdatedReg,
- SSAUpdater.getValueInMiddleOfBlock(&IMBB), Incoming.Reg);
- }
- } else {
- // The phi is not observed from outside a loop. Use a more accurate
- // lowering.
- PIA.analyze(MBB, Incomings);
-
- for (MachineBasicBlock *MBB : PIA.predecessors())
- SSAUpdater.addAvailableValue(
- MBB, insertUndefLaneMask(MBB, MRI, LaneMaskRegAttrs));
-
- for (auto &Incoming : Incomings) {
- MachineBasicBlock &IMBB = *Incoming.Block;
- if (PIA.isSource(IMBB)) {
- constrainAsLaneMask(Incoming);
- SSAUpdater.addAvailableValue(&IMBB, Incoming.Reg);
- } else {
- SSAUpdater.addUseBlock(&IMBB);
- Incoming.UpdatedReg = createLaneMaskReg(MRI, LaneMaskRegAttrs);
- SSAUpdater.addAvailableValue(&IMBB, Incoming.UpdatedReg);
- }
- }
-
- SSAUpdater.calculate();
-
- for (auto &Incoming : Incomings) {
- if (!Incoming.UpdatedReg.isValid())
- continue;
-
- MachineBasicBlock &IMBB = *Incoming.Block;
- buildMergeLaneMasks(
- IMBB, getSaluInsertionAtEnd(IMBB), {}, Incoming.UpdatedReg,
- SSAUpdater.getValueInMiddleOfBlock(&IMBB), Incoming.Reg);
- }
- }
+ mergeIncomingLaneMasks(DstReg, MBB, Incomings, SSAUpdater);
Register NewReg = SSAUpdater.getValueInMiddleOfBlock(&MBB);
if (NewReg != DstReg) {
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.h b/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
index 08423e925208f..8d5fcd79474cc 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/MachineSSAUpdater.h"
namespace llvm {
+class MachineIDFSSAUpdater;
namespace AMDGPU {
/// Incoming for lane mask phi as machine instruction, incoming register \p Reg
/// and incoming block \p Block are taken from machine instruction.
@@ -60,6 +61,15 @@ class PhiLoweringHelper {
public:
bool lowerPhis();
bool isConstantLaneMask(Register Reg, bool &Val) const;
+
+ /// Merge the \p Incomings lane masks into \p DstReg, the value owned by
+ /// \p MBB. Builds the per-predecessor merges and leaves \p SSAUpdater
+ /// calculated so the caller can query the merged value with
+ /// getValueInMiddleOfBlock. \p Incomings is sorted and its UpdatedReg fields
+ /// are filled in.
+ void mergeIncomingLaneMasks(Register DstReg, MachineBasicBlock &MBB,
+ SmallVectorImpl<Incoming> &Incomings,
+ MachineIDFSSAUpdater &SSAUpdater);
MachineBasicBlock::iterator
getSaluInsertionAtEnd(MachineBasicBlock &MBB) const;
More information about the llvm-commits
mailing list