[llvm] [AMDGPU] Fix wave.reduce.fsub with the DPP strategy (PR #228817)
Mian Miftah via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 20:44:13 PDT 2026
https://github.com/mmiftahx created https://github.com/llvm/llvm-project/pull/228817
On targets without DPP row broadcasts (GFX10+), the float DPP lowering
from #187214 combines the rows of each 32-lane half with ds_swizzle, and
the two halves of a wave64 with ds_permute. For float fsub these steps
used v_sub_f32, so partial sums were subtracted from each other before
the final negation: on gfx1201, lane values 1 to 32 in wave32 gave -256
instead of -528. Strategy 0, the default, takes this lowering too.
Use v_add_f32 for these steps, as the DPP row steps already do and as
integer sub does with S_ADD_I32. The negation at the end stays. GFX8 and
GFX9 use row_bcast with v_add_f32_dpp, and double fsub already adds, so
they don't change.
>From 21393df911210e37c0a1db20a56146cb0aa47fc8 Mon Sep 17 00:00:00 2001
From: mmiftahx <mmiftah.duna at gmail.com>
Date: Thu, 1 Oct 2026 21:35:11 -0500
Subject: [PATCH] [AMDGPU] Fix wave.reduce.fsub with the DPP strategy
On targets without DPP row broadcasts (GFX10+), the float DPP lowering
from #187214 combines the rows of each 32-lane half with ds_swizzle, and
the two halves of a wave64 with ds_permute. For float fsub these steps
used v_sub_f32, so partial sums were subtracted from each other before
the final negation: on gfx1201, lane values 1 to 32 in wave32 gave -256
instead of -528. Strategy 0, the default, takes this lowering too.
Use v_add_f32 for these steps, as the DPP row steps already do and as
integer sub does with S_ADD_I32. The negation at the end stays. GFX8 and
GFX9 use row_bcast with v_add_f32_dpp, and double fsub already adds, so
they don't change.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 ++
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll | 26 +++++++++----------
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4012992d3f1d6..7c5d8f6b5c02a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5995,6 +5995,8 @@ getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
llvm_unreachable("unhandled lane op");
}
unsigned ClampOpc = Opc;
+ if (Opc == AMDGPU::V_SUB_F32_e64)
+ ClampOpc = AMDGPU::V_ADD_F32_e64;
if (!ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
if (Opc == AMDGPU::S_SUB_I32)
ClampOpc = AMDGPU::S_ADD_I32;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index 2b3365dc49b94..255e802a10bd1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -1379,10 +1379,10 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1064DAGISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064DAGISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1064DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1064DAGISEL-NEXT: ds_permute_b32 v4, v5, v3
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1064DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1064DAGISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1064DAGISEL-NEXT: v_readlane_b32 s6, v3, 63
; GFX1064DAGISEL-NEXT: s_mov_b64 exec, s[4:5]
@@ -1419,10 +1419,10 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1064GISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064GISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1064GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1064GISEL-NEXT: ds_permute_b32 v4, v5, v3
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1064GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1064GISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1064GISEL-NEXT: v_readlane_b32 s6, v3, 63
; GFX1064GISEL-NEXT: s_mov_b64 exec, s[4:5]
@@ -1454,7 +1454,7 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1032DAGISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032DAGISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1032DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032DAGISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1032DAGISEL-NEXT: v_readlane_b32 s5, v3, 31
; GFX1032DAGISEL-NEXT: s_mov_b32 exec_lo, s4
@@ -1485,7 +1485,7 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1032GISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032GISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1032GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032GISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1032GISEL-NEXT: v_readlane_b32 s5, v3, 31
; GFX1032GISEL-NEXT: s_mov_b32 exec_lo, s4
@@ -1525,10 +1525,10 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1164DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1164DAGISEL-NEXT: ds_permute_b32 v4, v5, v3
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1164DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v3, 63
@@ -1570,10 +1570,10 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1164GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1164GISEL-NEXT: ds_permute_b32 v4, v5, v3
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1164GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v3, 63
@@ -1609,7 +1609,7 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1132DAGISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1132DAGISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1132DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1132DAGISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_readlane_b32 s1, v3, 31
@@ -1643,7 +1643,7 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1132GISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1132GISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX1132GISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1132GISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_readlane_b32 s1, v3, 31
@@ -1682,7 +1682,7 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX12DAGISEL-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX12DAGISEL-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
; GFX12DAGISEL-NEXT: s_wait_dscnt 0x0
-; GFX12DAGISEL-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12DAGISEL-NEXT: v_add_f32_e32 v3, v3, v4
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12DAGISEL-NEXT: v_sub_f32_e32 v3, 0, v3
; GFX12DAGISEL-NEXT: v_readlane_b32 s1, v3, 31
More information about the llvm-commits
mailing list