[llvm] [AMDGPU] Do not commute DPP instructions with a non-identity dpp_ctrl (PR #218393)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 01:49:23 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/218393
>From 0c3a9f2dac0f63303912689dd9496875a940852c Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 24 Aug 2026 14:47:51 +0200
Subject: [PATCH 1/3] [AMDGPU] Do not commute DPP instructions with a
non-identity dpp_ctrl
DPP only swizzles src0, and that does not move with the operands, so commuting changes which value each lane actually reads
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 1 +
llvm/test/CodeGen/AMDGPU/dpp_combine.ll | 16 ++++
.../CodeGen/AMDGPU/si-fold-operands-gfx11.mir | 89 ++++++++++++++++++-
4 files changed, 116 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index b2ae51a1b85f7..29371a71b7cda 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2898,11 +2898,21 @@ bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
return isImmOperandLegal(MI, OpIdx1, MO0);
}
+bool SIInstrInfo::isCommutableDPP(const MachineInstr &MI) const {
+ if (!isDPP(MI))
+ return true;
+ const MachineOperand *DppCtrl = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
+ return DppCtrl && DppCtrl->getImm() == AMDGPU::DPP::QUAD_PERM_ID;
+}
+
MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
unsigned Src0Idx,
unsigned Src1Idx) const {
assert(!NewMI && "this should never be used");
+ if (!isCommutableDPP(MI))
+ return nullptr;
+
unsigned Opc = MI.getOpcode();
int CommutedOpcode = commuteOpcode(Opc);
if (CommutedOpcode == -1)
@@ -2957,6 +2967,9 @@ MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
unsigned &SrcOpIdx0,
unsigned &SrcOpIdx1) const {
+ if (!isCommutableDPP(MI))
+ return false;
+
return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 6c7b2d7d2279e..2df9396cd557a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -234,6 +234,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
AMDGPU::OpName Src1OpName) const;
bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
unsigned toIdx) const;
+ bool isCommutableDPP(const MachineInstr &MI) const;
MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
unsigned OpIdx0,
unsigned OpIdx1) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/dpp_combine.ll b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
index 68d3bee1ac635..89fb18d8fcbc2 100644
--- a/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
@@ -228,6 +228,22 @@ define amdgpu_kernel void @dpp_src1_sgpr(ptr addrspace(1) %out, i16 %in) {
ret void
}
+; clamp forces VOP3 encoding so src1 takes an inline constant, but folding
+; 1.0 there needs a commute that would move row_shl:1 onto %x instead.
+; GCN-LABEL: {{^}}dpp_src1_imm_no_commute:
+; GFX9GFX10: v_mov_b32_dpp {{v[0-9]+}}, {{v[0-9]+}} row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9GFX10: v_add_f32_e64 v0, {{v[0-9]+}}, v0 clamp
+; GFX11-TRUE16: v_add_f32_e64_dpp v0, {{v[0-9]+}}, v0 clamp row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX11-FAKE16: v_add_f32_e64_dpp v0, {{v[0-9]+}}, v0 clamp row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+define float @dpp_src1_imm_no_commute(float %x) {
+entry:
+ %dpp = tail call float @llvm.amdgcn.update.dpp.f32(float 0.0, float 1.0, i32 257, i32 15, i32 15, i1 true)
+ %add = fadd float %dpp, %x
+ %mx = tail call float @llvm.maxnum.f32(float %add, float 0.0)
+ %mn = tail call float @llvm.minnum.f32(float %mx, float 1.0)
+ ret float %mn
+}
+
declare i32 @llvm.amdgcn.workitem.id.x()
declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0
declare float @llvm.ceil.f32(float)
diff --git a/llvm/test/CodeGen/AMDGPU/si-fold-operands-gfx11.mir b/llvm/test/CodeGen/AMDGPU/si-fold-operands-gfx11.mir
index 004accffea0fe..faded3fd7d346 100644
--- a/llvm/test/CodeGen/AMDGPU/si-fold-operands-gfx11.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-fold-operands-gfx11.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgpu11.00 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX_NO_SRC1_SGPR %s
-# RUN: llc -mtriple=amdgpu11.50 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX_SRC1_SGPR %s
-# RUN: llc -mtriple=amdgpu12.00 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX_SRC1_SGPR %s
+# RUN: llc -mtriple=amdgpu11.00 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX,GFX_NO_SRC1_SGPR %s
+# RUN: llc -mtriple=amdgpu11.50 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX,GFX_SRC1_SGPR %s
+# RUN: llc -mtriple=amdgpu12.00 -run-pass=si-fold-operands -o - %s | FileCheck -check-prefixes=GFX,GFX_SRC1_SGPR %s
# GFX_NO_SRC1_SGPR: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
@@ -24,3 +24,86 @@ body: |
%2:vgpr_32 = COPY %1:sreg_32
%3:vgpr_32, %4:sreg_32_xexec = V_ADD_CO_U32_e64_dpp %0:vgpr_32, %2:vgpr_32, %0:vgpr_32, 0, 228, 12, 15, 0, implicit $exec
...
+
+# a non-identity dpp_ctrl must block this commute (DPP is positional to src0).
+# GFX: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+# GFX: [[SGPR:%[0-9]+]]:sreg_32 = COPY $sgpr0
+# GFX: [[VCOPY:%[0-9]+]]:vgpr_32 = COPY [[SGPR]]
+# GFX: {{%[0-9]+}}:vgpr_32, {{%[0-9]+}}:sreg_32_xexec = V_ADD_CO_U32_e64_dpp [[VGPR]], [[VCOPY]], [[VGPR]], 0, 273, 12, 15, 0, implicit $exec
+---
+name: no_commute_sgpr_copy_to_dpp_src1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:sreg_32 = COPY $sgpr0
+
+ ; row_shr:1: must never combine
+ %2:vgpr_32 = COPY %1:sreg_32
+ %3:vgpr_32, %4:sreg_32_xexec = V_ADD_CO_U32_e64_dpp %0:vgpr_32, %2:vgpr_32, %0:vgpr_32, 0, 273, 12, 15, 0, implicit $exec
+...
+
+# same, but reaches commuteInstructionImpl via a different operand-swap path.
+# GFX: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+# GFX: [[MOV:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+# GFX: [[OLD:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# GFX: {{%[0-9]+}}:vgpr_32 = V_ADD_F32_e64_dpp [[OLD]], 0, [[MOV]], 0, [[VGPR]], 0, 0, 273, 12, 15, 0, implicit $mode, implicit $exec
+---
+name: no_commute_imm_to_dpp_src1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ %2:vgpr_32 = IMPLICIT_DEF
+
+ ; row_shr:1: must never combine
+ %3:vgpr_32 = V_ADD_F32_e64_dpp %2:vgpr_32, 0, %1:vgpr_32, 0, %0:vgpr_32, 0, 0, 273, 12, 15, 0, implicit $mode, implicit $exec
+...
+
+# identity quad_perm reads its own src0 per lane, so the commute stays legal.
+# GFX: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+# GFX_NO_SRC1_SGPR: [[MOV:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+# GFX_NO_SRC1_SGPR: [[OLD:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# GFX_NO_SRC1_SGPR: {{%[0-9]+}}:vgpr_32 = V_ADD_F32_e64_dpp [[OLD]], 0, [[MOV]], 0, [[VGPR]], 0, 0, 228, 12, 15, 0, implicit $mode, implicit $exec
+# GFX_SRC1_SGPR: [[OLD:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# GFX_SRC1_SGPR: {{%[0-9]+}}:vgpr_32 = V_ADD_F32_e64_dpp [[OLD]], 0, [[VGPR]], 0, 1065353216, 0, 0, 228, 12, 15, 0, implicit $mode, implicit $exec
+---
+name: commute_imm_to_dpp_src1_identity_quad_perm
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ %2:vgpr_32 = IMPLICIT_DEF
+
+ ; should be combined only on subtargets that allow an inline constant for src1
+ %3:vgpr_32 = V_ADD_F32_e64_dpp %2:vgpr_32, 0, %1:vgpr_32, 0, %0:vgpr_32, 0, 0, 228, 12, 15, 0, implicit $mode, implicit $exec
+...
+
+# in-place src1 folding needs no commute, so non-identity dpp_ctrl doesn't block it.
+# GFX: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+# GFX: [[SGPR:%[0-9]+]]:sreg_32 = COPY $sgpr0
+# GFX_NO_SRC1_SGPR: [[VCOPY:%[0-9]+]]:vgpr_32 = COPY [[SGPR]]
+# GFX_NO_SRC1_SGPR: {{%[0-9]+}}:vgpr_32, {{%[0-9]+}}:sreg_32_xexec = V_ADD_CO_U32_e64_dpp [[VGPR]], [[VGPR]], [[VCOPY]], 0, 273, 12, 15, 0, implicit $exec
+# GFX_SRC1_SGPR: {{%[0-9]+}}:vgpr_32, {{%[0-9]+}}:sreg_32_xexec = V_ADD_CO_U32_e64_dpp [[VGPR]], [[VGPR]], [[SGPR]], 0, 273, 12, 15, 0, implicit $exec
+---
+name: fold_sgpr_copy_to_dpp_src1_in_place
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:sreg_32 = COPY $sgpr0
+
+ ; should be combined only on subtargets that allow sgpr for src1
+ %2:vgpr_32 = COPY %1:sreg_32
+ %3:vgpr_32, %4:sreg_32_xexec = V_ADD_CO_U32_e64_dpp %0:vgpr_32, %0:vgpr_32, %2:vgpr_32, 0, 273, 12, 15, 0, implicit $exec
+...
>From dfa9cb49f9ea497c6feacf790d0ed55dff059259 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 25 Aug 2026 07:04:38 +0200
Subject: [PATCH 2/3] address the comments
---
llvm/test/CodeGen/AMDGPU/dpp_combine.ll | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/dpp_combine.ll b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
index 89fb18d8fcbc2..f1aafc891867e 100644
--- a/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
@@ -231,10 +231,8 @@ define amdgpu_kernel void @dpp_src1_sgpr(ptr addrspace(1) %out, i16 %in) {
; clamp forces VOP3 encoding so src1 takes an inline constant, but folding
; 1.0 there needs a commute that would move row_shl:1 onto %x instead.
; GCN-LABEL: {{^}}dpp_src1_imm_no_commute:
-; GFX9GFX10: v_mov_b32_dpp {{v[0-9]+}}, {{v[0-9]+}} row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX9GFX10: v_add_f32_e64 v0, {{v[0-9]+}}, v0 clamp
-; GFX11-TRUE16: v_add_f32_e64_dpp v0, {{v[0-9]+}}, v0 clamp row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX11-FAKE16: v_add_f32_e64_dpp v0, {{v[0-9]+}}, v0 clamp row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9GFX10: v_mov_b32_dpp [[V:v[0-9]+]], {{v[0-9]+}} row_shl:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9GFX10: v_add_f32_e64 [[V0:v[0-9]+]], [[V]], [[V0]] clamp
define float @dpp_src1_imm_no_commute(float %x) {
entry:
%dpp = tail call float @llvm.amdgcn.update.dpp.f32(float 0.0, float 1.0, i32 257, i32 15, i32 15, i1 true)
>From f73fb4bea214d5a159d304ad3859a8362368bac0 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 25 Aug 2026 10:48:59 +0200
Subject: [PATCH 3/3] Address comment
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 10 +++++-----
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 29371a71b7cda..d0675b367f183 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2898,11 +2898,11 @@ bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
return isImmOperandLegal(MI, OpIdx1, MO0);
}
-bool SIInstrInfo::isCommutableDPP(const MachineInstr &MI) const {
+bool SIInstrInfo::isNonCommutableDPP(const MachineInstr &MI) const {
if (!isDPP(MI))
- return true;
+ return false;
const MachineOperand *DppCtrl = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
- return DppCtrl && DppCtrl->getImm() == AMDGPU::DPP::QUAD_PERM_ID;
+ return !DppCtrl || DppCtrl->getImm() != AMDGPU::DPP::QUAD_PERM_ID;
}
MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
@@ -2910,7 +2910,7 @@ MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
unsigned Src1Idx) const {
assert(!NewMI && "this should never be used");
- if (!isCommutableDPP(MI))
+ if (isNonCommutableDPP(MI))
return nullptr;
unsigned Opc = MI.getOpcode();
@@ -2967,7 +2967,7 @@ MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
unsigned &SrcOpIdx0,
unsigned &SrcOpIdx1) const {
- if (!isCommutableDPP(MI))
+ if (isNonCommutableDPP(MI))
return false;
return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2df9396cd557a..188d2225494a3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -234,7 +234,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
AMDGPU::OpName Src1OpName) const;
bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
unsigned toIdx) const;
- bool isCommutableDPP(const MachineInstr &MI) const;
+ bool isNonCommutableDPP(const MachineInstr &MI) const;
MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
unsigned OpIdx0,
unsigned OpIdx1) const override;
More information about the llvm-commits
mailing list