[llvm-branch-commits] [llvm] [AMDGPU] Lower uniform uaddsat.i16 to SALU instructions (PR #210156)
Anshil Gandhi via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 3 21:44:21 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/210156
>From 74ccc44d4a0ef854ccd0da4615482412a134b47f Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Wed, 22 Jul 2026 12:11:30 -0500
Subject: [PATCH] [AMDGPU] Lower uniform uaddsat.i16 to SALU instructions
Promote uniform i16 uadd.sat to i32 in promoteUniformOpToI32 so it
lowers to SALU (s_add_i32 + s_min_u32) instead of VALU + readfirstlane.
The saturating add on zero-extended operands reduces to
umin(add(lhs, rhs), 0xffff).
Co-authored-by: Cursor <cursoragent at cursor.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +-
llvm/test/CodeGen/AMDGPU/uaddsat.ll | 124 +++++++++---------
3 files changed, 75 insertions(+), 62 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 962988ff97e39..7fa4b9a09aea9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1059,6 +1059,7 @@ bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
case ISD::UMIN:
case ISD::UMAX:
case ISD::USUBSAT:
+ case ISD::UADDSAT:
if (isTypeLegal(MVT::i16) &&
(!DestVT.isVector() ||
!isOperationLegal(ISD::ADD, MVT::v2i16))) { // Check if VOP3P
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 612342b159583..0642b90c330b9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1090,6 +1090,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::UMIN,
ISD::UMAX,
ISD::USUBSAT,
+ ISD::UADDSAT,
ISD::AND,
ISD::OR,
ISD::XOR,
@@ -8927,6 +8928,7 @@ static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
case ISD::UMIN:
case ISD::UMAX:
case ISD::USUBSAT:
+ case ISD::UADDSAT:
return ISD::ZERO_EXTEND;
case ISD::ADD:
case ISD::SUB:
@@ -8976,7 +8978,7 @@ SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
- Opc == ISD::USUBSAT);
+ Opc == ISD::USUBSAT || Opc == ISD::UADDSAT);
EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
: Op->getOperand(0).getValueType();
@@ -9018,7 +9020,12 @@ SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
SDValue NewVal;
if (Opc == ISD::SELECT)
NewVal = DAG.getNode(ISD::SELECT, DL, ExtTy, {Op->getOperand(0), LHS, RHS});
- else
+ else if (Opc == ISD::UADDSAT) {
+ SDValue Sum = DAG.getNode(ISD::ADD, DL, ExtTy, LHS, RHS);
+ SDValue MaxVal = DAG.getConstant(
+ APInt::getMaxValue(OpTy.getScalarSizeInBits()).zext(32), DL, ExtTy);
+ NewVal = DAG.getNode(ISD::UMIN, DL, ExtTy, Sum, MaxVal);
+ } else
NewVal = DAG.getNode(Opc, DL, ExtTy, {LHS, RHS});
return DAG.getZExtOrTrunc(NewVal, DL, OpTy);
@@ -18869,6 +18876,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::UMIN:
case ISD::UMAX:
case ISD::USUBSAT:
+ case ISD::UADDSAT:
if (auto Res = promoteUniformOpToI32(SDValue(N, 0), DCI))
return Res;
break;
diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
index b5e9611e15c27..e188bb032b029 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
@@ -73,37 +73,36 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
;
; GFX8-LABEL: s_uaddsat_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_add_i32 s0, s0, s1
+; GFX8-NEXT: s_min_u32 s0, s0, 0xffff
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_add_u16_e64 v0, s0, v0 clamp
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_min_u32 s0, s0, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_uaddsat_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_add_nc_u16 v0, s0, s1 clamp
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_min_u32 s0, s0, 0xffff
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_uaddsat_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_uaddsat_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, s0, s1 clamp
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_uaddsat_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
ret i16 %result
}
@@ -125,15 +124,16 @@ define amdgpu_ps <2 x i16> @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u16_e64 v1, s0, v1 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX8-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX8-NEXT: s_lshr_b32 s1, s1, 16
+; GFX8-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-NEXT: s_add_i32 s0, s0, s1
+; GFX8-NEXT: s_add_i32 s3, s3, s2
+; GFX8-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX8-NEXT: s_min_u32 s2, s3, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_or_b32 s0, s2, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v2i16:
@@ -180,18 +180,20 @@ define amdgpu_ps <3 x i16> @s_uaddsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v3i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s2, 16
-; GFX8-NEXT: s_lshr_b32 s5, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u16_e64 v2, s0, v2 clamp
-; GFX8-NEXT: v_add_u16_e64 v1, s1, v1 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-NEXT: s_add_i32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX8-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX8-NEXT: s_lshr_b32 s2, s2, 16
+; GFX8-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-NEXT: s_add_i32 s0, s0, s2
+; GFX8-NEXT: s_add_i32 s4, s4, s3
+; GFX8-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX8-NEXT: s_min_u32 s3, s4, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_min_u32 s1, s1, 0xffff
+; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v3i16:
@@ -251,24 +253,26 @@ define amdgpu_ps <4 x i16> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_lshr_b32 s4, s2, 16
-; GFX8-NEXT: s_lshr_b32 s5, s0, 16
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_add_u16_sdwa v1, v2, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_add_u16_e64 v2, s1, v2 clamp
-; GFX8-NEXT: v_add_u16_e64 v3, s0, v3 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v1
-; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_and_b32 s4, s3, 0xffff
+; GFX8-NEXT: s_and_b32 s5, s1, 0xffff
+; GFX8-NEXT: s_add_i32 s5, s5, s4
+; GFX8-NEXT: s_lshr_b32 s3, s3, 16
+; GFX8-NEXT: s_lshr_b32 s1, s1, 16
+; GFX8-NEXT: s_min_u32 s4, s5, 0xffff
+; GFX8-NEXT: s_add_i32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX8-NEXT: s_and_b32 s5, s0, 0xffff
+; GFX8-NEXT: s_lshr_b32 s2, s2, 16
+; GFX8-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-NEXT: s_add_i32 s0, s0, s2
+; GFX8-NEXT: s_min_u32 s1, s1, 0xffff
+; GFX8-NEXT: s_add_i32 s5, s5, s3
+; GFX8-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX8-NEXT: s_min_u32 s3, s5, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: s_or_b32 s1, s4, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v4i16:
More information about the llvm-branch-commits
mailing list