[llvm] 014a6cd - [AMDGPU][GlobalISel] Add RegBankLegalize support for G_AMDGPU_MAD_U64_U32 (#176060)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 19 11:21:49 PST 2026
Author: vangthao95
Date: 2026-01-19T11:21:45-08:00
New Revision: 014a6cdbb347bd8c436e132d55cbd682ba4d15fa
URL: https://github.com/llvm/llvm-project/commit/014a6cdbb347bd8c436e132d55cbd682ba4d15fa
DIFF: https://github.com/llvm/llvm-project/commit/014a6cdbb347bd8c436e132d55cbd682ba4d15fa.diff
LOG: [AMDGPU][GlobalISel] Add RegBankLegalize support for G_AMDGPU_MAD_U64_U32 (#176060)
Also add partial G_MUL support in the form of uniform S32 rule for G_MUL
since lowering G_AMDGPU_MAD_U64_U32 requires both uniform and divergent
S32 G_MUL.
Added:
llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index de0e501e21a06..9f6a9c317667a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -19,6 +19,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineUniformityAnalysis.h"
@@ -716,6 +717,53 @@ bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
return true;
}
+bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
+ Register Dst0 = MI.getOperand(0).getReg();
+ Register Dst1 = MI.getOperand(1).getReg();
+ Register Src0 = MI.getOperand(2).getReg();
+ Register Src1 = MI.getOperand(3).getReg();
+ Register Src2 = MI.getOperand(4).getReg();
+
+ const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
+
+ // Keep the multiplication on the SALU.
+ Register DstLo = B.buildMul(SgprRB_S32, Src0, Src1).getReg(0);
+ Register DstHi = MRI.createVirtualRegister(SgprRB_S32);
+ if (ST.hasScalarMulHiInsts()) {
+ B.buildInstr(AMDGPU::G_UMULH, {{DstHi}}, {Src0, Src1});
+ } else {
+ auto VSrc0 = B.buildCopy(VgprRB_S32, Src0);
+ auto VSrc1 = B.buildCopy(VgprRB_S32, Src1);
+ auto MulHi = B.buildInstr(AMDGPU::G_UMULH, {VgprRB_S32}, {VSrc0, VSrc1});
+ buildReadAnyLane(B, DstHi, MulHi.getReg(0), RBI);
+ }
+
+ // Accumulate and produce the "carry-out" bit.
+
+ // The "carry-out" is defined as bit 64 of the result when computed as a
+ // big integer. For unsigned multiply-add, this matches the usual
+ // definition of carry-out.
+ if (mi_match(Src2, MRI, MIPatternMatch::m_ZeroInt())) {
+ // No accumulate: result is just the multiplication, carry is 0.
+ B.buildMergeLikeInstr(Dst0, {DstLo, DstHi});
+ B.buildConstant(Dst1, 0);
+ } else {
+ // Accumulate: add Src2 to the multiplication result with carry chain.
+ Register Src2Lo = MRI.createVirtualRegister(SgprRB_S32);
+ Register Src2Hi = MRI.createVirtualRegister(SgprRB_S32);
+ B.buildUnmerge({Src2Lo, Src2Hi}, Src2);
+
+ auto AddLo = B.buildUAddo(SgprRB_S32, SgprRB_S32, DstLo, Src2Lo);
+ auto AddHi =
+ B.buildUAdde(SgprRB_S32, SgprRB_S32, DstHi, Src2Hi, AddLo.getReg(1));
+ B.buildMergeLikeInstr(Dst0, {AddLo.getReg(0), AddHi.getReg(0)});
+ B.buildCopy(Dst1, AddHi.getReg(1));
+ }
+
+ MI.eraseFromParent();
+ return true;
+}
+
bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
@@ -857,6 +905,8 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerV_BFE(MI);
case S_BFE:
return lowerS_BFE(MI);
+ case UniMAD64:
+ return lowerUniMAD64(MI);
case SplitTo32:
return lowerSplitTo32(MI);
case SplitTo32Select:
@@ -1204,9 +1254,11 @@ bool RegBankLegalizeHelper::applyMappingDst(
assert(RB == SgprRB);
Register NewDst = MRI.createVirtualRegister(VccRB_S1);
Op.setReg(NewDst);
- auto CopyS32_Vcc =
- B.buildInstr(AMDGPU::G_AMDGPU_COPY_SCC_VCC, {SgprRB_S32}, {NewDst});
- B.buildTrunc(Reg, CopyS32_Vcc);
+ if (!MRI.use_empty(Reg)) {
+ auto CopyS32_Vcc =
+ B.buildInstr(AMDGPU::G_AMDGPU_COPY_SCC_VCC, {SgprRB_S32}, {NewDst});
+ B.buildTrunc(Reg, CopyS32_Vcc);
+ }
break;
}
case UniInVgprS16: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 1dc0278d6d90d..f92ed3de6cf27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -125,6 +125,7 @@ class RegBankLegalizeHelper {
bool lowerUnpackBitShift(MachineInstr &MI);
bool lowerV_BFE(MachineInstr &MI);
bool lowerS_BFE(MachineInstr &MI);
+ bool lowerUniMAD64(MachineInstr &MI);
bool lowerSplitTo32(MachineInstr &MI);
bool lowerSplitTo16(MachineInstr &MI);
bool lowerSplitTo32Select(MachineInstr &MI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index da0acf83b6955..d22e54c1b41fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -495,7 +495,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
- addRulesForGOpcs({G_MUL}, Standard).Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}});
+ addRulesForGOpcs({G_MUL}, Standard)
+ .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
+ .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}});
bool hasMulHi = ST->hasScalarMulHiInsts();
addRulesForGOpcs({G_UMULH, G_SMULH}, Standard)
@@ -503,6 +505,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasMulHi)
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasMulHi);
+ addRulesForGOpcs({G_AMDGPU_MAD_U64_U32}, Standard)
+ .Div(S64, {{Vgpr64, Vcc}, {Vgpr32, Vgpr32, Vgpr64}})
+ .Uni(S64, {{Sgpr64, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr64}, UniMAD64});
+
addRulesForGOpcs({G_XOR, G_OR, G_AND}, StandardB)
.Any({{UniS1}, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
.Any({{DivS1}, {{Vcc}, {Vcc, Vcc}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 7381cd8f1aac4..371e91ceb2649 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -225,6 +225,7 @@ enum LoweringMethodID {
S_BFE,
V_BFE,
VgprToVccCopy,
+ UniMAD64,
SplitTo32,
ScalarizeToS16,
SplitTo32Select,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
new file mode 100644
index 0000000000000..060a5d92b3fd7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -o - %s | FileCheck --check-prefix=GFX7 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -o - %s | FileCheck --check-prefix=GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -o - %s | FileCheck --check-prefix=GFX11 %s
+
+define amdgpu_ps void @mad_i64_uniform(i64 inreg %a, i64 inreg %b, ptr addrspace(1) %out) {
+; GFX7-LABEL: mad_i64_uniform:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mul_hi_u32 v2, s0, v2
+; GFX7-NEXT: s_mul_i32 s4, s0, s2
+; GFX7-NEXT: s_mul_i32 s0, s0, s3
+; GFX7-NEXT: s_mul_i32 s1, s1, s2
+; GFX7-NEXT: v_readfirstlane_b32 s5, v2
+; GFX7-NEXT: s_add_u32 s0, s0, s5
+; GFX7-NEXT: s_add_u32 s5, s1, s0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX7-NEXT: s_endpgm
+;
+; GFX90A-LABEL: mad_i64_uniform:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_mul_i32 s4, s0, s2
+; GFX90A-NEXT: s_mul_hi_u32 s5, s0, s2
+; GFX90A-NEXT: s_mul_i32 s0, s0, s3
+; GFX90A-NEXT: s_mul_i32 s1, s1, s2
+; GFX90A-NEXT: s_add_u32 s0, s1, s0
+; GFX90A-NEXT: s_add_i32 s5, s5, s0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX90A-NEXT: s_endpgm
+;
+; GFX11-LABEL: mad_i64_uniform:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mul_hi_u32 s4, s0, s2
+; GFX11-NEXT: s_mul_i32 s3, s0, s3
+; GFX11-NEXT: s_mul_i32 s1, s1, s2
+; GFX11-NEXT: s_add_i32 s3, s4, s3
+; GFX11-NEXT: s_mul_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s3, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: s_endpgm
+ %result = mul i64 %a, %b
+ store i64 %result, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @mad_u64_u32_uniform_carry(i32 inreg %a, i32 inreg %b, i64 inreg %c, ptr addrspace(1) %out) {
+; GFX7-LABEL: mad_u64_u32_uniform_carry:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
+; GFX7-NEXT: v_mul_hi_u32 v2, s0, v2
+; GFX7-NEXT: s_mul_i32 s0, s0, s1
+; GFX7-NEXT: s_add_u32 s0, s0, s2
+; GFX7-NEXT: v_readfirstlane_b32 s1, v2
+; GFX7-NEXT: s_addc_u32 s1, s1, s3
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX7-NEXT: s_endpgm
+;
+; GFX90A-LABEL: mad_u64_u32_uniform_carry:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_mul_i32 s4, s0, s1
+; GFX90A-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX90A-NEXT: s_add_u32 s0, s4, s2
+; GFX90A-NEXT: s_addc_u32 s1, s1, s3
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX90A-NEXT: s_endpgm
+;
+; GFX11-LABEL: mad_u64_u32_uniform_carry:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mul_i32 s4, s0, s1
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_add_u32 s0, s4, s2
+; GFX11-NEXT: s_addc_u32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: s_endpgm
+ %a.ext = zext i32 %a to i64
+ %b.ext = zext i32 %b to i64
+ %mul = mul i64 %a.ext, %b.ext
+ %result = add i64 %mul, %c
+ store i64 %result, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @mad_i64_div(i64 %a, i64 %b, ptr addrspace(1) %out) {
+; GFX7-LABEL: mad_i64_div:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v0, v2, 0
+; GFX7-NEXT: v_mad_u64_u32 v[9:10], s[0:1], v0, v3, v[7:8]
+; GFX7-NEXT: v_mad_u64_u32 v[7:8], s[0:1], v1, v2, v[9:10]
+; GFX7-NEXT: flat_store_dwordx2 v[4:5], v[6:7]
+; GFX7-NEXT: s_endpgm
+;
+; GFX90A-LABEL: mad_i64_div:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v0, v3, 0
+; GFX90A-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v0, v2, 0
+; GFX90A-NEXT: v_mad_u64_u32 v[10:11], s[0:1], v1, v2, v[8:9]
+; GFX90A-NEXT: v_add_u32_e32 v7, v7, v10
+; GFX90A-NEXT: global_store_dwordx2 v[4:5], v[6:7], off
+; GFX90A-NEXT: s_endpgm
+;
+; GFX11-LABEL: mad_i64_div:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, v0, v2, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mad_u64_u32 v[9:10], null, v0, v3, v[7:8]
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, v1, v2, v[9:10]
+; GFX11-NEXT: global_store_b64 v[4:5], v[6:7], off
+; GFX11-NEXT: s_endpgm
+ %result = mul i64 %a, %b
+ store i64 %result, ptr addrspace(1) %out
+ ret void
+}
More information about the llvm-commits
mailing list