[llvm-branch-commits] [llvm] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8 accumulator (PR #225322)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 22 01:32:29 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225322
>From da5b743dc74a4d6aebc6b95b72c36a425269b7ef Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 22 Sep 2026 16:00:29 +0800
Subject: [PATCH] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8
accumulator
Fold a constant add into the accumulator operand of sudot4 and sudot8 when
clamping is disabled:
```
sudot(a, b, C1, false) + C2 -> sudot(a, b, C1 + C2, false)
```
Subtraction by a constant is canonicalized to addition of its negation.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 26 ++++++++++++++++
.../InstCombine/AMDGPU/llvm.amdgcn.sudot.ll | 30 +++++++------------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 073682c526e7e..b08a5a708e4d6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -2087,6 +2087,32 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
}
return std::nullopt;
}
+ case Intrinsic::amdgcn_sudot4:
+ case Intrinsic::amdgcn_sudot8: {
+ // Reassociating across a saturating accumulate is not valid.
+ if (!II.hasOneUse() || !match(II.getArgOperand(5), m_Zero()))
+ break;
+
+ const APInt *Acc;
+ if (!match(II.getArgOperand(4), m_APInt(Acc)))
+ break;
+
+ auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
+ if (!AccumUser)
+ break;
+
+ const APInt *AccumDelta;
+ Constant *NewAcc;
+ if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
+ NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
+ else
+ break;
+
+ IC.replaceOperand(II, 4, NewAcc);
+ IC.replaceInstUsesWith(*AccumUser, &II);
+ IC.eraseInstFromFunction(*AccumUser);
+ return &II;
+ }
case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
Value *Src0 = II.getArgOperand(0);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
index a9840e1c7e489..97a32749dac39 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll
@@ -4,8 +4,7 @@
define i32 @sudot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false)
@@ -16,8 +15,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) {
define i32 @sudot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false)
@@ -105,8 +103,7 @@ define i32 @sudot4_all_constants_clamp() {
define i32 @sudot4_zero_acc_add_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -129,8 +126,7 @@ define i32 @sudot4_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) {
define i32 @sudot4_zero_acc_add_const_lhs(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const_lhs(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -157,8 +153,7 @@ define i32 @sudot4_zero_acc_multiple_users(i32 %a, i32 %b) {
define i32 @sudot4_zero_acc_sub_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot4_zero_acc_sub_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 -9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false)
@@ -181,8 +176,7 @@ define i32 @sudot4_zero_acc_const_sub(i32 %a, i32 %b) {
define i32 @sudot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false)
@@ -238,8 +232,7 @@ define i32 @sudot8_all_constants_clamp() {
define i32 @sudot8_zero_acc_add_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -262,8 +255,7 @@ define i32 @sudot8_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) {
define i32 @sudot8_zero_acc_add_const_lhs(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const_lhs(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -290,8 +282,7 @@ define i32 @sudot8_zero_acc_multiple_users(i32 %a, i32 %b) {
define i32 @sudot8_zero_acc_sub_const(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_zero_acc_sub_const(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 -9, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false)
@@ -314,8 +305,7 @@ define i32 @sudot8_zero_acc_const_sub(i32 %a, i32 %b) {
define i32 @sudot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sudot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false)
More information about the llvm-branch-commits
mailing list