[llvm] [AMDGPU][InstCombine] Fold zero dot operands to accumulator (PR #225003)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 18:49:38 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225003
>From 11612ad1892bfba3fb45d927a7f746e4b43117fc Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 21 Sep 2026 13:19:30 +0800
Subject: [PATCH] [AMDGPU][InstCombine] Fold zero dot operands to accumulator
Fold AMDGPU dot intrinsics when either operand is zero.
`dot(a, 0) = 0` and `dot(0, b) = 0`, so replace the intrinsic with its accumulator.
This avoids unrelated clamp and add/sub reassociation cases.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 3 ++
.../InstCombine/AMDGPU/llvm.amdgcn.dot.ll | 45 +++++++------------
2 files changed, 18 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 06d2b84411604..7721114fdd2f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -2014,6 +2014,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return &II;
}
+ if (match(Src1, m_Zero()))
+ return IC.replaceInstUsesWith(II, II.getArgOperand(2));
+
if (Instruction *I = foldConstantIntoDotAccumulator(II, 2, 3, IC))
return I;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
index 1e495e38cc57c..f6f22d57ae51b 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll
@@ -26,8 +26,7 @@ define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @sdot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @sdot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[B]], <2 x i16> zeroinitializer, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -37,8 +36,7 @@ define i32 @sdot2_a_zero(<2 x i16> %b) {
define i32 @sdot2_const_lhs(<2 x i16> %b, i32 %acc) {
; CHECK-LABEL: define i32 @sdot2_const_lhs(
; CHECK-SAME: <2 x i16> [[B:%.*]], i32 [[ACC:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[B]], <2 x i16> zeroinitializer, i32 [[ACC]], i1 false)
-; CHECK-NEXT: ret i32 [[DOT]]
+; CHECK-NEXT: ret i32 [[ACC]]
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 %acc, i1 false)
ret i32 %dot
@@ -47,8 +45,7 @@ define i32 @sdot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @sdot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @sdot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -129,8 +126,7 @@ define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) {
define i32 @udot2_a_zero(<2 x i16> %b) {
; CHECK-LABEL: define i32 @udot2_a_zero(
; CHECK-SAME: <2 x i16> [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[B]], <2 x i16> zeroinitializer, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -140,8 +136,7 @@ define i32 @udot2_a_zero(<2 x i16> %b) {
define i32 @udot2_const_lhs(<2 x i16> %b, i32 %acc) {
; CHECK-LABEL: define i32 @udot2_const_lhs(
; CHECK-SAME: <2 x i16> [[B:%.*]], i32 [[ACC:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[B]], <2 x i16> zeroinitializer, i32 [[ACC]], i1 false)
-; CHECK-NEXT: ret i32 [[DOT]]
+; CHECK-NEXT: ret i32 [[ACC]]
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 %acc, i1 false)
ret i32 %dot
@@ -150,8 +145,7 @@ define i32 @udot2_const_lhs(<2 x i16> %b, i32 %acc) {
define i32 @udot2_b_zero(<2 x i16> %a) {
; CHECK-LABEL: define i32 @udot2_b_zero(
; CHECK-SAME: <2 x i16> [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -217,8 +211,7 @@ define i32 @sdot4_sub(i32 %a, i32 %b) {
define i32 @sdot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[B]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 0, i32 %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -238,8 +231,7 @@ define i32 @sdot4_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 0, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -376,8 +368,7 @@ define i32 @udot4_sub(i32 %a, i32 %b) {
define i32 @udot4_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot4_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[B]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot4(i32 0, i32 %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -397,8 +388,7 @@ define i32 @udot4_const_lhs(i32 %b, i32 %acc) {
define i32 @udot4_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot4_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 0, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -453,8 +443,7 @@ define i32 @sdot8_sub(i32 %a, i32 %b) {
define i32 @sdot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @sdot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[B]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 0, i32 %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -474,8 +463,7 @@ define i32 @sdot8_const_lhs(i32 %b, i32 %acc) {
define i32 @sdot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @sdot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 0, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -530,8 +518,7 @@ define i32 @udot8_sub(i32 %a, i32 %b) {
define i32 @udot8_a_zero(i32 %b) {
; CHECK-LABEL: define i32 @udot8_a_zero(
; CHECK-SAME: i32 [[B:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[B]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 7, i1 false)
%r = add i32 %dot, 9
@@ -541,8 +528,7 @@ define i32 @udot8_a_zero(i32 %b) {
define i32 @udot8_const_lhs(i32 %b, i32 %acc) {
; CHECK-LABEL: define i32 @udot8_const_lhs(
; CHECK-SAME: i32 [[B:%.*]], i32 [[ACC:%.*]]) {
-; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[B]], i32 0, i32 [[ACC]], i1 false)
-; CHECK-NEXT: ret i32 [[DOT]]
+; CHECK-NEXT: ret i32 [[ACC]]
;
%dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 %acc, i1 false)
ret i32 %dot
@@ -551,8 +537,7 @@ define i32 @udot8_const_lhs(i32 %b, i32 %acc) {
define i32 @udot8_b_zero(i32 %a) {
; CHECK-LABEL: define i32 @udot8_b_zero(
; CHECK-SAME: i32 [[A:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 16, i1 false)
-; CHECK-NEXT: ret i32 [[R]]
+; CHECK-NEXT: ret i32 16
;
%dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 0, i32 7, i1 false)
%r = add i32 %dot, 9
More information about the llvm-commits
mailing list