[llvm] [AMDGPU][InstCombine] Fold constant add/sub into the dot accumulator (PR #225002)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 01:07:22 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225002
>From d1af1cb3cf45d713519b45e7c3c62b19f4bdfe18 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 21 Sep 2026 12:15:13 +0800
Subject: [PATCH 1/2] [AMDGPU][InstCombine] Fold constant add/sub into the dot
accumulator
`amdgcn.{s,u}dot{2,4,8}(a, b, C) +/- K -> dot(a, b, C +/- K)` when both the
accumulator C and K are constants. The new constant is computed with
wrapping APInt arithmetic to match the non-clamping accumulate.
Only applies when clamp is false (the saturating accumulate does not
reassociate) and the dot has a single use. K - dot is left alone since it
would need the dot product negated.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 38 +++++++
.../InstCombine/AMDGPU/llvm.amdgcn.dot.ll | 99 +++++++------------
2 files changed, 71 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 073682c526e7e..dcbdde6ccbdfa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1968,6 +1968,44 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven);
return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result));
}
+ case Intrinsic::amdgcn_sdot2:
+ case Intrinsic::amdgcn_udot2:
+ case Intrinsic::amdgcn_sdot4:
+ case Intrinsic::amdgcn_udot4:
+ case Intrinsic::amdgcn_sdot8:
+ case Intrinsic::amdgcn_udot8: {
+ if (!match(II.getArgOperand(3), m_Zero()) || !II.hasOneUse())
+ break;
+
+ const APInt *Acc;
+ if (!match(II.getArgOperand(2), m_APInt(Acc)))
+ break;
+
+ auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
+ if (!AccumUser)
+ break;
+
+ unsigned Opcode = AccumUser->getOpcode();
+ if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+ break;
+
+ // C - dot cannot be folded without negating the dot product.
+ if (Opcode == Instruction::Sub && AccumUser->getOperand(0) != &II)
+ break;
+
+ const APInt *AccumDelta;
+ Value *ConstOp =
+ AccumUser->getOperand(AccumUser->getOperand(0) == &II ? 1 : 0);
+ if (!match(ConstOp, m_APInt(AccumDelta)))
+ break;
+
+ Constant *NewAcc = ConstantInt::get(II.getType(), Opcode == Instruction::Add
+ ? *Acc + *AccumDelta
+ : *Acc - *AccumDelta);
+ IC.replaceInstUsesWith(*AccumUser, &II);
+ IC.eraseInstFromFunction(*AccumUser);
+ return IC.replaceOperand(II, 2, NewAcc);
+ }
case Intrinsic::amdgcn_fmul_legacy: {
Value *Op0 = II.getArgOperand(0);
Value *Op1 = II.getArgOperand(1);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
index 05c8111537cd9..a719483577d8d 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
@@ -4,8 +4,7 @@
define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_add(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -16,8 +15,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_sub(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -28,8 +26,7 @@ define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
@@ -50,8 +47,7 @@ define i32 @sdot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @sdot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @sdot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
@@ -62,8 +58,7 @@ define i32 @sdot2_b_zero(<2 x i16> %a) {
define i32 @sdot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_const_nonzero(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
@@ -112,8 +107,7 @@ define i32 @sdot2_multi_use(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_add(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -124,8 +118,7 @@ define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_sub(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -136,8 +129,7 @@ define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
@@ -158,8 +150,7 @@ define i32 @udot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @udot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @udot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
@@ -170,8 +161,7 @@ define i32 @udot2_b_zero(<2 x i16> %a) {
define i32 @udot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_const_nonzero(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
@@ -194,8 +184,7 @@ define i32 @udot2_clamp(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -206,8 +195,7 @@ define i32 @sdot4_add(i32 %a, i32 %b) {
define i32 @sdot4_add_commuted(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add_commuted(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -218,8 +206,7 @@ define i32 @sdot4_add_commuted(i32 %a, i32 %b) {
define i32 @sdot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -230,8 +217,7 @@ define i32 @sdot4_sub(i32 %a, i32 %b) {
define i32 @sdot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 0, i32 %b, i32 7, i1 false)
@@ -252,8 +238,7 @@ define i32 @sdot4_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 0, i32 7, i1 false)
@@ -264,8 +249,7 @@ define i32 @sdot4_b_zero(i32 %a) {
define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_sub_to_zero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 9, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 0, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 9, i1 false)
@@ -276,8 +260,7 @@ define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) {
define i32 @sdot4_add_chain(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add_chain(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 6
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 13, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -289,8 +272,7 @@ define i32 @sdot4_add_chain(i32 %a, i32 %b) {
define i32 @sdot4_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false)
@@ -372,8 +354,7 @@ define void @sdot4_no_use(i32 %a, i32 %b) {
define i32 @udot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -384,8 +365,7 @@ define i32 @udot4_add(i32 %a, i32 %b) {
define i32 @udot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -396,8 +376,7 @@ define i32 @udot4_sub(i32 %a, i32 %b) {
define i32 @udot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 0, i32 %b, i32 7, i1 false)
@@ -418,8 +397,7 @@ define i32 @udot4_const_lhs(i32 %b, i32 %acc) {
define i32 @udot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 0, i32 7, i1 false)
@@ -430,8 +408,7 @@ define i32 @udot4_b_zero(i32 %a) {
define i32 @udot4_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false)
@@ -454,8 +431,7 @@ define i32 @udot4_clamp(i32 %a, i32 %b) {
define i32 @sdot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -466,8 +442,7 @@ define i32 @sdot8_add(i32 %a, i32 %b) {
define i32 @sdot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -478,8 +453,7 @@ define i32 @sdot8_sub(i32 %a, i32 %b) {
define i32 @sdot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 0, i32 %b, i32 7, i1 false)
@@ -500,8 +474,7 @@ define i32 @sdot8_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 0, i32 7, i1 false)
@@ -512,8 +485,7 @@ define i32 @sdot8_b_zero(i32 %a) {
define i32 @sdot8_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false)
@@ -536,8 +508,7 @@ define i32 @sdot8_clamp(i32 %a, i32 %b) {
define i32 @udot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -548,8 +519,7 @@ define i32 @udot8_add(i32 %a, i32 %b) {
define i32 @udot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -560,8 +530,7 @@ define i32 @udot8_sub(i32 %a, i32 %b) {
define i32 @udot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 7, i1 false)
@@ -582,8 +551,7 @@ define i32 @udot8_const_lhs(i32 %b, i32 %acc) {
define i32 @udot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 0, i32 7, i1 false)
@@ -594,8 +562,7 @@ define i32 @udot8_b_zero(i32 %a) {
define i32 @udot8_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false)
>From dc8b3ba5ab2c7efa85d214ec90e6ef0e66bc1d4f Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 22 Sep 2026 16:05:21 +0800
Subject: [PATCH 2/2] Update code
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 18 ++++--------------
1 file changed, 4 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index dcbdde6ccbdfa..e5e25bebf8fce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1985,23 +1985,13 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
if (!AccumUser)
break;
- unsigned Opcode = AccumUser->getOpcode();
- if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
- break;
-
- // C - dot cannot be folded without negating the dot product.
- if (Opcode == Instruction::Sub && AccumUser->getOperand(0) != &II)
- break;
-
const APInt *AccumDelta;
- Value *ConstOp =
- AccumUser->getOperand(AccumUser->getOperand(0) == &II ? 1 : 0);
- if (!match(ConstOp, m_APInt(AccumDelta)))
+ Constant *NewAcc;
+ if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta))))
+ NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta);
+ else
break;
- Constant *NewAcc = ConstantInt::get(II.getType(), Opcode == Instruction::Add
- ? *Acc + *AccumDelta
- : *Acc - *AccumDelta);
IC.replaceInstUsesWith(*AccumUser, &II);
IC.eraseInstFromFunction(*AccumUser);
return IC.replaceOperand(II, 2, NewAcc);
More information about the llvm-commits
mailing list