[llvm-branch-commits] [llvm] [AMDGPU][InstCombine] Fold constant add/sub into the dot accumulator (PR #225002)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Sep 21 06:43:07 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225002
>From cfae0666afc5840e6a808320eb85dd58107827a2 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 21 Sep 2026 12:15:13 +0800
Subject: [PATCH] [AMDGPU][InstCombine] Fold constant add/sub into the dot
accumulator
`amdgcn.{s,u}dot{2,4,8}(a, b, C) +/- K -> dot(a, b, C +/- K)` when both the
accumulator C and K are constants. The new constant is computed with
wrapping APInt arithmetic to match the non-clamping accumulate.
Only applies when clamp is false (the saturating accumulate does not
reassociate) and the dot has a single use. K - dot is left alone since it
would need the dot product negated.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 38 +++++++
.../InstCombine/AMDGPU/llvm.amdgcn.dot.ll | 99 +++++++------------
2 files changed, 71 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 073682c526e7e..dcbdde6ccbdfa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1968,6 +1968,44 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven);
return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result));
}
+ case Intrinsic::amdgcn_sdot2:
+ case Intrinsic::amdgcn_udot2:
+ case Intrinsic::amdgcn_sdot4:
+ case Intrinsic::amdgcn_udot4:
+ case Intrinsic::amdgcn_sdot8:
+ case Intrinsic::amdgcn_udot8: {
+ if (!match(II.getArgOperand(3), m_Zero()) || !II.hasOneUse())
+ break;
+
+ const APInt *Acc;
+ if (!match(II.getArgOperand(2), m_APInt(Acc)))
+ break;
+
+ auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
+ if (!AccumUser)
+ break;
+
+ unsigned Opcode = AccumUser->getOpcode();
+ if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+ break;
+
+ // C - dot cannot be folded without negating the dot product.
+ if (Opcode == Instruction::Sub && AccumUser->getOperand(0) != &II)
+ break;
+
+ const APInt *AccumDelta;
+ Value *ConstOp =
+ AccumUser->getOperand(AccumUser->getOperand(0) == &II ? 1 : 0);
+ if (!match(ConstOp, m_APInt(AccumDelta)))
+ break;
+
+ Constant *NewAcc = ConstantInt::get(II.getType(), Opcode == Instruction::Add
+ ? *Acc + *AccumDelta
+ : *Acc - *AccumDelta);
+ IC.replaceInstUsesWith(*AccumUser, &II);
+ IC.eraseInstFromFunction(*AccumUser);
+ return IC.replaceOperand(II, 2, NewAcc);
+ }
case Intrinsic::amdgcn_fmul_legacy: {
Value *Op0 = II.getArgOperand(0);
Value *Op1 = II.getArgOperand(1);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
index 68ba966ea3b95..909e2edc238bd 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
@@ -4,8 +4,7 @@
define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_add(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -16,8 +15,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_sub(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -28,8 +26,7 @@ define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
@@ -50,8 +47,7 @@ define i32 @sdot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @sdot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @sdot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
@@ -62,8 +58,7 @@ define i32 @sdot2_b_zero(<2 x i16> %a) {
define i32 @sdot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_const_nonzero(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
@@ -112,8 +107,7 @@ define i32 @sdot2_multi_use(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_add(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -124,8 +118,7 @@ define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_sub(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
@@ -136,8 +129,7 @@ define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
@@ -158,8 +150,7 @@ define i32 @udot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @udot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @udot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
@@ -170,8 +161,7 @@ define i32 @udot2_b_zero(<2 x i16> %a) {
define i32 @udot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_const_nonzero(
; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false)
@@ -194,8 +184,7 @@ define i32 @udot2_clamp(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -206,8 +195,7 @@ define i32 @sdot4_add(i32 %a, i32 %b) {
define i32 @sdot4_add_commuted(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add_commuted(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -218,8 +206,7 @@ define i32 @sdot4_add_commuted(i32 %a, i32 %b) {
define i32 @sdot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -230,8 +217,7 @@ define i32 @sdot4_sub(i32 %a, i32 %b) {
define i32 @sdot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 0, i32 %b, i32 7, i1 false)
@@ -252,8 +238,7 @@ define i32 @sdot4_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 0, i32 7, i1 false)
@@ -264,8 +249,7 @@ define i32 @sdot4_b_zero(i32 %a) {
define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_sub_to_zero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 9, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 0, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 9, i1 false)
@@ -276,8 +260,7 @@ define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) {
define i32 @sdot4_add_chain(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_add_chain(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 6
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 13, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -289,8 +272,7 @@ define i32 @sdot4_add_chain(i32 %a, i32 %b) {
define i32 @sdot4_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot4_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false)
@@ -363,8 +345,7 @@ define i32 @sdot4_add_x(i32 %a, i32 %b, i32 %x) {
define i32 @udot4_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -375,8 +356,7 @@ define i32 @udot4_add(i32 %a, i32 %b) {
define i32 @udot4_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false)
@@ -387,8 +367,7 @@ define i32 @udot4_sub(i32 %a, i32 %b) {
define i32 @udot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 0, i32 %b, i32 7, i1 false)
@@ -409,8 +388,7 @@ define i32 @udot4_const_lhs(i32 %b, i32 %acc) {
define i32 @udot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 0, i32 7, i1 false)
@@ -421,8 +399,7 @@ define i32 @udot4_b_zero(i32 %a) {
define i32 @udot4_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot4_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false)
@@ -445,8 +422,7 @@ define i32 @udot4_clamp(i32 %a, i32 %b) {
define i32 @sdot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -457,8 +433,7 @@ define i32 @sdot8_add(i32 %a, i32 %b) {
define i32 @sdot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -469,8 +444,7 @@ define i32 @sdot8_sub(i32 %a, i32 %b) {
define i32 @sdot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 0, i32 %b, i32 7, i1 false)
@@ -491,8 +465,7 @@ define i32 @sdot8_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 0, i32 7, i1 false)
@@ -503,8 +476,7 @@ define i32 @sdot8_b_zero(i32 %a) {
define i32 @sdot8_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @sdot8_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false)
@@ -527,8 +499,7 @@ define i32 @sdot8_clamp(i32 %a, i32 %b) {
define i32 @udot8_add(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_add(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -539,8 +510,7 @@ define i32 @udot8_add(i32 %a, i32 %b) {
define i32 @udot8_sub(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_sub(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 6, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false)
@@ -551,8 +521,7 @@ define i32 @udot8_sub(i32 %a, i32 %b) {
define i32 @udot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 7, i1 false)
@@ -573,8 +542,7 @@ define i32 @udot8_const_lhs(i32 %b, i32 %acc) {
define i32 @udot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 0, i32 7, i1 false)
@@ -585,8 +553,7 @@ define i32 @udot8_b_zero(i32 %a) {
define i32 @udot8_const_nonzero(i32 %a, i32 %b) {
; CHECK-LABEL: define i32 @udot8_const_nonzero(
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false)
-; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 16, i1 false)
; CHECK-NEXT: ret i32 [[R]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false)
More information about the llvm-branch-commits
mailing list