[llvm] [AMDGPU] Update LiveVariables in SILowerControlFlow::combineMasks (PR #217844)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 01:48:00 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/217844
>From c24414c5a9301893747be0ddce235e17f39a535b Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 21 Aug 2026 10:29:13 +0200
Subject: [PATCH 1/3] [AMDGPU] Update LiveVariables in
SILowerControlFlow::combineMasks
---
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 39 +++++--
...wer-control-flow-live-variables-update.mir | 109 ++++++++++++++++++
2 files changed, 140 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 25694a53f35dd2..3842c8a6bb5964 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -104,11 +104,12 @@ class SILowerControlFlow {
void findMaskOperands(MachineInstr &MI, unsigned OpNo,
SmallVectorImpl<MachineOperand *> &Src) const;
- void combineMasks(MachineInstr &MI);
+ void combineMasks(MachineInstr &MI, MachineBasicBlock::iterator &OuterNext);
bool removeMBBifRedundant(MachineBasicBlock &MBB);
- MachineBasicBlock *process(MachineInstr &MI);
+ MachineBasicBlock *process(MachineInstr &MI,
+ MachineBasicBlock::iterator &OuterNext);
// Skip to the next instruction, ignoring debug instructions, and trivial
// block boundaries (blocks that have one (typically fallthrough) successor,
@@ -601,7 +602,8 @@ void SILowerControlFlow::findMaskOperands(
// S_AND_B64 x, (S_AND_B64 x, y) => S_AND_B64 x, y
// S_OR_B64 x, (S_OR_B64 x, y) => S_OR_B64 x, y
// One of the operands is exec mask.
-void SILowerControlFlow::combineMasks(MachineInstr &MI) {
+void SILowerControlFlow::combineMasks(MachineInstr &MI,
+ MachineBasicBlock::iterator &OuterNext) {
assert(MI.getNumExplicitOperands() == 3);
SmallVector<MachineOperand *, 2> Src1, Src2;
findMaskOperands(MI, 1, Src1);
@@ -636,10 +638,29 @@ void SILowerControlFlow::combineMasks(MachineInstr &MI) {
return;
Register Reg = MI.getOperand(OpToReplace).getReg();
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
MI.removeOperand(OpToReplace);
MI.addOperand(*KeepOp);
- if (MRI->use_empty(Reg))
- MRI->getUniqueVRegDef(Reg)->eraseFromParent();
+
+ // The fold moves the last use of Reg and of the Def sources onto MI.
+ SmallSet<Register, 4> RecomputeLV;
+ if (LV) {
+ RecomputeLV.insert(Reg);
+ for (const MachineOperand &Op : Def->all_uses())
+ if (Op.getReg().isVirtual())
+ RecomputeLV.insert(Op.getReg());
+ }
+
+ if (MRI->use_empty(Reg)) {
+ if (OuterNext == Def->getIterator())
+ ++OuterNext;
+ Def->eraseFromParent();
+ }
+
+ if (LV)
+ for (Register R : RecomputeLV)
+ if (!MRI->def_empty(R)) // Skip Reg if its def was just erased.
+ LV->recomputeForSingleDefVirtReg(R);
}
void SILowerControlFlow::optimizeEndCf() {
@@ -676,7 +697,9 @@ void SILowerControlFlow::optimizeEndCf() {
}
}
-MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
+MachineBasicBlock *
+SILowerControlFlow::process(MachineInstr &MI,
+ MachineBasicBlock::iterator &OuterNext) {
MachineBasicBlock &MBB = *MI.getParent();
MachineBasicBlock::iterator I(MI);
MachineInstr *Prev = (I != MBB.begin()) ? &*(std::prev(I)) : nullptr;
@@ -723,7 +746,7 @@ MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
case AMDGPU::S_AND_B32:
case AMDGPU::S_OR_B32:
// Cleanup bit manipulations on exec mask
- combineMasks(MaskMI);
+ combineMasks(MaskMI, OuterNext);
break;
default:
I = MBB.end();
@@ -839,7 +862,7 @@ bool SILowerControlFlow::run(MachineFunction &MF) {
case AMDGPU::SI_WATERFALL_LOOP:
case AMDGPU::SI_LOOP:
case AMDGPU::SI_END_CF:
- SplitMBB = process(MI);
+ SplitMBB = process(MI, Next);
Changed = true;
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
index 32746ef5fce9e8..0a4caba006cf6f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
@@ -260,3 +260,112 @@ body: |
S_BRANCH %bb.2
...
+
+# combineMasks folds the inner S_AND away, moving the kill of %0 to the outer.
+---
+name: live_variables_update_combine_masks
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: live_variables_update_combine_masks
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY killed $sgpr4
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed [[DEF]], implicit-def $scc
+ ; CHECK-NEXT: S_NOP 0
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 64, killed [[COPY]], implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr0 = COPY killed [[S_AND_B32_]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed $sgpr0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr4
+
+ %0:sreg_32 = COPY killed $sgpr4
+ %1:sreg_64_xexec = IMPLICIT_DEF
+
+ bb.1:
+ SI_END_CF killed %1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+ S_NOP 0
+ %2:sreg_32 = S_AND_B32 %0, %0, implicit-def dead $scc
+ %3:sreg_32 = S_AND_B32 64, %2, implicit-def dead $scc
+ $sgpr0 = COPY killed %3
+ S_ENDPGM 0, implicit $sgpr0
+
+...
+
+# The folded-away def is the instruction the lowering loop is about to visit.
+---
+name: live_variables_update_combine_masks_dead_def_is_next
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: live_variables_update_combine_masks_dead_def_is_next
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY killed $sgpr4
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed [[DEF]], implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 64, killed [[COPY]], implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr0 = COPY killed [[S_AND_B32_]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed $sgpr0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr4
+
+ %0:sreg_32 = COPY killed $sgpr4
+ %1:sreg_64_xexec = IMPLICIT_DEF
+
+ bb.1:
+ SI_END_CF killed %1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+ %2:sreg_32 = S_AND_B32 %0, %0, implicit-def dead $scc
+ %3:sreg_32 = S_AND_B32 64, %2, implicit-def dead $scc
+ $sgpr0 = COPY killed %3
+ S_ENDPGM 0, implicit $sgpr0
+
+...
+
+# The inner S_AND survives here, so its stale kill of %0 must be cleared.
+---
+name: live_variables_update_combine_masks_def_survives
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: live_variables_update_combine_masks_def_survives
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY killed $sgpr4
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed [[DEF]], implicit-def $scc
+ ; CHECK-NEXT: S_NOP 0
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 64, killed [[COPY]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 killed [[S_AND_B32_]], killed [[S_AND_B32_1]], implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr0 = COPY killed [[S_XOR_B32_]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed $sgpr0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr4
+
+ %0:sreg_32 = COPY killed $sgpr4
+ %1:sreg_64_xexec = IMPLICIT_DEF
+
+ bb.1:
+ SI_END_CF killed %1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+ S_NOP 0
+ %2:sreg_32 = S_AND_B32 %0, %0, implicit-def dead $scc
+ %3:sreg_32 = S_AND_B32 64, %2, implicit-def dead $scc
+ %4:sreg_32 = S_XOR_B32 %2, %3, implicit-def dead $scc
+ $sgpr0 = COPY killed %4
+ S_ENDPGM 0, implicit $sgpr0
+
+...
>From 1edfeb53571cfdae4065d5825362fba04fd40866 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 21 Aug 2026 11:25:26 +0200
Subject: [PATCH 2/3] Address comments
---
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 6 ++--
...wer-control-flow-live-variables-update.mir | 35 +++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 3842c8a6bb5964..a291e6262f91a2 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -657,10 +657,12 @@ void SILowerControlFlow::combineMasks(MachineInstr &MI,
Def->eraseFromParent();
}
- if (LV)
- for (Register R : RecomputeLV)
+ if (LV) {
+ for (Register R : RecomputeLV) {
if (!MRI->def_empty(R)) // Skip Reg if its def was just erased.
LV->recomputeForSingleDefVirtReg(R);
+ }
+ }
}
void SILowerControlFlow::optimizeEndCf() {
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
index 0a4caba006cf6f..5d11687ae23583 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
@@ -369,3 +369,38 @@ body: |
S_ENDPGM 0, implicit $sgpr0
...
+
+# The operand being replaced is marked undef; the fold must still erase
+# the dead inner S_AND without leaving stale kill flags.
+---
+name: live_variables_update_combine_masks_undef_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: live_variables_update_combine_masks_undef_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY killed $sgpr4
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed [[DEF]], implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 64, killed [[COPY]], implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr0 = COPY killed [[S_AND_B32_]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed $sgpr0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr4
+
+ %0:sreg_32 = COPY killed $sgpr4
+ %1:sreg_64_xexec = IMPLICIT_DEF
+
+ bb.1:
+ SI_END_CF killed %1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+ %2:sreg_32 = S_AND_B32 %0, %0, implicit-def dead $scc
+ %3:sreg_32 = S_AND_B32 64, undef %2, implicit-def dead $scc
+ $sgpr0 = COPY killed %3
+ S_ENDPGM 0, implicit $sgpr0
+
+...
>From fed2f3913e0b709f38098ff9318d536506ce6f5b Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 21 Aug 2026 12:15:27 +0200
Subject: [PATCH 3/3] assert + revert test
---
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 1 +
...wer-control-flow-live-variables-update.mir | 35 -------------------
2 files changed, 1 insertion(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index a291e6262f91a2..f345016af8dfc2 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -639,6 +639,7 @@ void SILowerControlFlow::combineMasks(MachineInstr &MI,
Register Reg = MI.getOperand(OpToReplace).getReg();
MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ assert(Def);
MI.removeOperand(OpToReplace);
MI.addOperand(*KeepOp);
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
index 5d11687ae23583..0a4caba006cf6f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
@@ -369,38 +369,3 @@ body: |
S_ENDPGM 0, implicit $sgpr0
...
-
-# The operand being replaced is marked undef; the fold must still erase
-# the dead inner S_AND without leaving stale kill flags.
----
-name: live_variables_update_combine_masks_undef_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: live_variables_update_combine_masks_undef_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY killed $sgpr4
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed [[DEF]], implicit-def $scc
- ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 64, killed [[COPY]], implicit-def dead $scc
- ; CHECK-NEXT: $sgpr0 = COPY killed [[S_AND_B32_]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit killed $sgpr0
- bb.0:
- successors: %bb.1
- liveins: $sgpr4
-
- %0:sreg_32 = COPY killed $sgpr4
- %1:sreg_64_xexec = IMPLICIT_DEF
-
- bb.1:
- SI_END_CF killed %1, implicit-def $exec, implicit-def dead $scc, implicit $exec
- %2:sreg_32 = S_AND_B32 %0, %0, implicit-def dead $scc
- %3:sreg_32 = S_AND_B32 64, undef %2, implicit-def dead $scc
- $sgpr0 = COPY killed %3
- S_ENDPGM 0, implicit $sgpr0
-
-...
More information about the llvm-commits
mailing list