[llvm] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8 accumulator (PR #225322)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 08:54:00 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225322
>From 0fcfc1cfda5ad1bfbf5a3da80319f077630b9912 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 22 Sep 2026 16:00:29 +0800
Subject: [PATCH 1/3] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8
accumulator
Fold a constant add into the accumulator operand of sudot4 and sudot8 when
clamping is disabled:
```
sudot(a, b, C1, false) + C2 -> sudot(a, b, C1 + C2, false)
```
Subtraction by a constant is canonicalized to addition of its negation.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 26 ++++++++++++++++
.../InstCombine/AMDGPU/llvm.amdgcn.sudot.ll | 30 +++++++------------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 073682c526e7e7..b08a5a708e4d64 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -2087,6 +2087,32 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
}
return std::nullopt;
}
+ case Intrinsic::amdgcn_sudot4:
+ case Intrinsic::amdgcn_sudot8: {
+ // Reassociating across a saturating accumulate is not valid.
+ if (!II.hasOneUse() || !match(II.getArgOperand(5), m_Zero()))
+ break;
+
+ const APInt *Acc;
+ if (!match(II.getArgOperand(4), m_APInt(Acc)))
+ break;
+
+ auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
+ if (!AccumUser)
+ break;
+
+ const APInt *AccumDelta;
+ Constant *NewAcc;
+ if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
+ NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
+ else
+ break;
+
+ IC.replaceOperand(II, 4, NewAcc);
+ IC.replaceInstUsesWith(*AccumUser, &II);
+ IC.eraseInstFromFunction(*AccumUser);
+ return &II;
+ }
case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
Value *Src0 = II.getArgOperand(0);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
index a9840e1c7e4897..97a32749dac397 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
@@ -4,8 +4,7 @@
define i32 @sudot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false)
@@ -16,8 +15,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) {
define i32 @sudot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false)
@@ -105,8 +103,7 @@ define i32 @sudot4_all_constants_clamp() {
define i32 @sudot4_zero_acc_add_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -129,8 +126,7 @@ define i32 @sudot4_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) {
define i32 @sudot4_zero_acc_add_const_lhs(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const_lhs(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -157,8 +153,7 @@ define i32 @sudot4_zero_acc_multiple_users(i32 %a, i32 %b) {
define i32 @sudot4_zero_acc_sub_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_sub_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 -9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -181,8 +176,7 @@ define i32 @sudot4_zero_acc_const_sub(i32 %a, i32 %b) {
define i32 @sudot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false)
@@ -238,8 +232,7 @@ define i32 @sudot8_all_constants_clamp() {
define i32 @sudot8_zero_acc_add_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -262,8 +255,7 @@ define i32 @sudot8_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) {
define i32 @sudot8_zero_acc_add_const_lhs(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const_lhs(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -290,8 +282,7 @@ define i32 @sudot8_zero_acc_multiple_users(i32 %a, i32 %b) {
define i32 @sudot8_zero_acc_sub_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_sub_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 -9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -314,8 +305,7 @@ define i32 @sudot8_zero_acc_const_sub(i32 %a, i32 %b) {
define i32 @sudot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false)
>From bc2e715dacb6cccac9f21a830ba53b6b11ba59b5 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 22 Sep 2026 20:04:21 +0800
Subject: [PATCH 2/3] Update for comments
---
llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index b08a5a708e4d64..d72c2bffaeb9e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -2103,11 +2103,11 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
const APInt *AccumDelta;
Constant *NewAcc;
- if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
- NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
- else
+ if (!match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
break;
+ NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
+
IC.replaceOperand(II, 4, NewAcc);
IC.replaceInstUsesWith(*AccumUser, &II);
IC.eraseInstFromFunction(*AccumUser);
>From db16db9eadaf17baa5e47b4b798bd61269cc92fc Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 22 Sep 2026 21:54:53 +0800
Subject: [PATCH 3/3] Update again
---
llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index d72c2bffaeb9e7..2df64afb94266a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -2102,11 +2102,10 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
break;
const APInt *AccumDelta;
- Constant *NewAcc;
if (!match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
break;
- NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
+ Constant *NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
IC.replaceOperand(II, 4, NewAcc);
IC.replaceInstUsesWith(*AccumUser, &II);
More information about the llvm-commits
mailing list