[llvm] [AMDGPU] Do not fold add chains wider than 32 bits into dot4 (PR #216015)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 08:32:40 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/216015
>From 1f2586966e185b1239720c592d0af7fc1289e707 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 13 Aug 2026 13:31:30 +0200
Subject: [PATCH 1/2] [AMDGPU] Do not fold add chains wider than 32 bits into
dot4
dot4 always produces a 32-bit result, so folding a wider accumulator would silently drop its upper bits
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
llvm/test/CodeGen/AMDGPU/idot4-test.ll | 66 +++++++++++++++++++++++
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6cbf4833fe66f..f25c90770d8bd 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -17738,7 +17738,9 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
return Folded;
}
- if ((isMul(LHS) || isMul(RHS)) && Subtarget->hasDot7Insts() &&
+ // dot4 produces a 32-bit result, so a wider VT can't be folded.
+ if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
+ (isMul(LHS) || isMul(RHS)) && Subtarget->hasDot7Insts() &&
(Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
SDValue TempNode(N, 0);
std::optional<bool> IsSigned;
diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
index d055059751205..dc3e836354eeb 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
@@ -1166,6 +1166,72 @@ define i32 @dot4_and_mask10_chain_not_add(i32 %x, i32 %y) {
ret i32 %result
}
+;------------------------------------------------------------------------------
+; NEGATIVE TESTS: ACCUMULATOR WIDER THAN A DOT4 CAN REPRESENT
+;------------------------------------------------------------------------------
+
+; i64 accumulator would lose its upper half if folded into a 32-bit dot4.
+define i64 @dot4_acc64_not_dot4(i8 %a0, i8 %a1, i8 %b0, i8 %b1, i64 %acc) {
+; GFX9-DL-LABEL: dot4_acc64_not_dot4:
+; GFX9-DL: ; %bb.0:
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_add_co_u32_e32 v1, vcc, v1, v4
+; GFX9-DL-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
+; GFX9-DL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-DL-NEXT: v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
+; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-DL-LABEL: dot4_acc64_not_dot4:
+; GFX10-DL: ; %bb.0:
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_hi_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_add_co_u32 v2, vcc_lo, v6, v4
+; GFX10-DL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo
+; GFX10-DL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-DL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v7, v1, vcc_lo
+; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: dot4_acc64_not_dot4:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_mul_hi_i32_i24_sdwa v9, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v8, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[8:9], 0, v[4:5]
+; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[6:7], 0, v[0:1]
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-NODL-LABEL: dot4_acc64_not_dot4:
+; GFX9-NODL: ; %bb.0:
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NODL-NEXT: v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_add_co_u32_e32 v1, vcc, v1, v4
+; GFX9-NODL-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
+; GFX9-NODL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NODL-NEXT: v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
+; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
+ %cv1e0 = sext i8 %a0 to i64
+ %cv1e1 = sext i8 %a1 to i64
+ %cv2e0 = sext i8 %b0 to i64
+ %cv2e1 = sext i8 %b1 to i64
+ %mul0 = mul nsw i64 %cv1e0, %cv2e0
+ %mul1 = mul nsw i64 %cv1e1, %cv2e1
+ %add = add i64 %mul1, %acc
+ %result = add i64 %mul0, %add
+ ret i64 %result
+}
+
declare i32 @llvm.sadd.sat.i32(i32, i32)
declare i32 @llvm.uadd.sat.i32(i32, i32)
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
>From a4e5c3849fd0d3fb6afa3eebb5ae8efd3c22260e Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 28 Aug 2026 17:32:29 +0200
Subject: [PATCH 2/2] decapsify
---
llvm/test/CodeGen/AMDGPU/idot4-test.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
index dc3e836354eeb..79017f3ea3cd7 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
@@ -1167,7 +1167,7 @@ define i32 @dot4_and_mask10_chain_not_add(i32 %x, i32 %y) {
}
;------------------------------------------------------------------------------
-; NEGATIVE TESTS: ACCUMULATOR WIDER THAN A DOT4 CAN REPRESENT
+; Negative tests: accumulator wider than a dot4 can represent
;------------------------------------------------------------------------------
; i64 accumulator would lose its upper half if folded into a 32-bit dot4.
More information about the llvm-commits
mailing list