[llvm] [AMDGPU] Fix Inefficient S_CSELECT_B64 Sequence (PR #167780)
Patrick Simmons via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 11:52:14 PDT 2026
https://github.com/linuxrocks123 updated https://github.com/llvm/llvm-project/pull/167780
>From d65a3ca292d84a5fedeafe15c98111b496adae8d Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Wed, 12 Nov 2025 14:18:26 -0500
Subject: [PATCH 1/5] Initial commit
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 39 +++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 8c45017cd4c43..de4291caac4d7 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -24,6 +24,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include <optional>
@@ -66,6 +67,7 @@ class SIPeepholeSDWA {
MachineInstr *createSDWAVersion(MachineInstr &MI);
bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
+ bool strengthReduceCSelect64(MachineFunction &MF);
public:
bool run(MachineFunction &MF);
@@ -1359,6 +1361,40 @@ void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
}
}
+bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : make_early_inc_range(MBB)) {
+ if (MI.getOpcode() != AMDGPU::S_CSELECT_B64)
+ continue;
+
+ Register Reg = MI.getOperand(0).getReg();
+ MachineInstr *MustBeVCNDMASK = MRI->getOneNonDBGUser(Reg);
+ if (!MustBeVCNDMASK ||
+ MustBeVCNDMASK->getOpcode() != AMDGPU::V_CNDMASK_B32_e64 ||
+ !MustBeVCNDMASK->getOperand(1).isImm() ||
+ !MustBeVCNDMASK->getOperand(2).isImm())
+ continue;
+
+ MachineInstr *MustBeVREADFIRSTLANE =
+ MRI->getOneNonDBGUser(MustBeVCNDMASK->getOperand(0).getReg());
+ if (!MustBeVREADFIRSTLANE ||
+ MustBeVREADFIRSTLANE->getOpcode() != AMDGPU::V_READFIRSTLANE_B32)
+ continue;
+
+ BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::S_CSELECT_B32),
+ MustBeVREADFIRSTLANE->getOperand(0).getReg())
+ .addImm(MI.getOperand(1).getImm())
+ .addImm(MI.getOperand(2).getImm())
+ .addReg(AMDGPU::SCC, RegState::Implicit);
+
+ MustBeVREADFIRSTLANE->eraseFromParent();
+ }
+
+ return Changed;
+}
+
bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -1433,6 +1469,9 @@ bool SIPeepholeSDWA::run(MachineFunction &MF) {
} while (Changed);
}
+ // Other target-specific SSA-form peephole optimizations
+ Ret |= strengthReduceCSelect64(MF);
+
return Ret;
}
>From 5f026cd98059eedc8d3c179b445f2a9729336a2a Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Wed, 12 Nov 2025 14:36:38 -0500
Subject: [PATCH 2/5] Fix
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index de4291caac4d7..7246a69edd587 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -1366,7 +1366,9 @@ bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF)
for (MachineInstr &MI : make_early_inc_range(MBB)) {
- if (MI.getOpcode() != AMDGPU::S_CSELECT_B64)
+ if (MI.getOpcode() != AMDGPU::S_CSELECT_B64 ||
+ !MI.getOperand(1).isImm() || !MI.getOperand(2).isImm() ||
+ (MI.getOperand(1).getImm() != 0 && MI.getOperand(2).getImm() != 0))
continue;
Register Reg = MI.getOperand(0).getReg();
@@ -1383,10 +1385,13 @@ bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
MustBeVREADFIRSTLANE->getOpcode() != AMDGPU::V_READFIRSTLANE_B32)
continue;
+ unsigned CSelectZeroOpIdx = MI.getOperand(1).getImm() ? 2 : 1;
+
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::S_CSELECT_B32),
MustBeVREADFIRSTLANE->getOperand(0).getReg())
- .addImm(MI.getOperand(1).getImm())
- .addImm(MI.getOperand(2).getImm())
+ .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx == 1 ? 2 : 1)
+ .getImm())
+ .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx).getImm())
.addReg(AMDGPU::SCC, RegState::Implicit);
MustBeVREADFIRSTLANE->eraseFromParent();
>From b2e51b6d0a88a63b9c7ec99b1427abab4161738c Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Wed, 12 Nov 2025 14:56:17 -0500
Subject: [PATCH 3/5] This is backwards
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 7246a69edd587..2b4409b11a319 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -1389,9 +1389,10 @@ bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::S_CSELECT_B32),
MustBeVREADFIRSTLANE->getOperand(0).getReg())
- .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx == 1 ? 2 : 1)
- .getImm())
- .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx).getImm())
+ .addImm(
+ MustBeVCNDMASK->getOperand((CSelectZeroOpIdx == 1 ? 2 : 1) + 2)
+ .getImm())
+ .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx + 2).getImm())
.addReg(AMDGPU::SCC, RegState::Implicit);
MustBeVREADFIRSTLANE->eraseFromParent();
>From a004846df67d479b138095a65d419106e2e99cb1 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Wed, 12 Nov 2025 18:02:36 -0500
Subject: [PATCH 4/5] This is forwards.
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 2b4409b11a319..7846101b9abcb 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -1389,10 +1389,10 @@ bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::S_CSELECT_B32),
MustBeVREADFIRSTLANE->getOperand(0).getReg())
+ .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx + 2).getImm())
.addImm(
MustBeVCNDMASK->getOperand((CSelectZeroOpIdx == 1 ? 2 : 1) + 2)
.getImm())
- .addImm(MustBeVCNDMASK->getOperand(CSelectZeroOpIdx + 2).getImm())
.addReg(AMDGPU::SCC, RegState::Implicit);
MustBeVREADFIRSTLANE->eraseFromParent();
>From ce658987f5f41fa9ee45585709f403047bf4a7be Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Wed, 17 Jun 2026 17:02:45 -0500
Subject: [PATCH 5/5] Review Bugfix
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 7846101b9abcb..650788d6780db 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -1368,6 +1368,7 @@ bool SIPeepholeSDWA::strengthReduceCSelect64(MachineFunction &MF) {
for (MachineInstr &MI : make_early_inc_range(MBB)) {
if (MI.getOpcode() != AMDGPU::S_CSELECT_B64 ||
!MI.getOperand(1).isImm() || !MI.getOperand(2).isImm() ||
+ (MI.getOperand(1).getImm() != 0 && MI.getOperand(1).getImm() != -1) ||
(MI.getOperand(1).getImm() != 0 && MI.getOperand(2).getImm() != 0))
continue;
More information about the llvm-commits
mailing list