[llvm] [AMDGPU] Keep divergent i64 mul feeding an add for the mad64 fold (PR #226963)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 04:25:22 PDT 2026
https://github.com/skc7 created https://github.com/llvm/llvm-project/pull/226963
Changes:
- `AMDGPUCodeGenPrepare` rewrites a divergent i64 mul with 24-bit operands into `llvm.amdgcn.mul.[iu]24.i64`, and narrows other i64 muls to an i32 `mul` plus `zext` when the product fits.
- Both hide the `ISD::MUL` that `SITargetLowering::tryFoldToMad64_32` matches. An `add` of the product is then lowered as a `mul24` pair plus an `add/addc` carry chain instead of a single `v_mad_[iu]64_[iu]32`.
- On GFX9+, skip both rewrites for a divergent i64 mul whose only user is an add, so the `SelectionDAG` add combine can form the mad.
>From c5bab5864e5a50b024d4d3b99a40a0559feeee4c Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 28 Sep 2026 16:51:45 +0530
Subject: [PATCH] [AMDGPU] Keep divergent i64 mul feeding an add for the mad64
fold
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 24 +++-
.../amdgpu-codegenprepare-mul24-mad64.ll | 84 ++++++++++++
.../atomic_optimizations_global_pointer.ll | 18 +--
.../atomic_optimizations_local_pointer.ll | 14 +-
llvm/test/CodeGen/AMDGPU/idot4-test.ll | 54 ++++----
llvm/test/CodeGen/AMDGPU/mad_u64_u32_mul24.ll | 123 ++++++++++++++++++
6 files changed, 267 insertions(+), 50 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24-mad64.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/mad_u64_u32_mul24.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 60fb91d58d2bc..8cae9c1d4e09a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -173,6 +173,10 @@ class AMDGPUCodeGenPrepareImpl
/// SelectionDAG has an issue where an and asserting the bits are known
bool replaceMulWithMul24(BinaryOperator &I) const;
+ /// \returns true if \p I should stay a plain i64 mul so SelectionDAG can
+ /// fold it into v_mad_[iu]64_[iu]32.
+ bool shouldKeepMulForMad64(const BinaryOperator &I) const;
+
/// Perform same function as equivalently named function in DAGCombiner. Since
/// we expand some divisions here, we need to perform this before obscuring.
bool foldBinOpIntoSelect(BinaryOperator &I) const;
@@ -1446,14 +1450,26 @@ bool AMDGPUCodeGenPrepareImpl::tryNarrowMathIfNoOverflow(Instruction *I) {
return true;
}
+// Mul24 or narrowing would hide the ISD::MUL that tryFoldToMad64_32 matches.
+bool AMDGPUCodeGenPrepareImpl::shouldKeepMulForMad64(
+ const BinaryOperator &I) const {
+ if (I.getOpcode() != Instruction::Mul || !I.getType()->isIntegerTy(64))
+ return false;
+ if (ST.getGeneration() < AMDGPUSubtarget::GFX9 || UA.isUniformAtDef(&I))
+ return false;
+ return I.hasOneUse() && match(I.user_back(), m_Add(m_Value(), m_Value()));
+}
+
bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
if (foldBinOpIntoSelect(I))
return true;
- if (UseMul24Intrin && replaceMulWithMul24(I))
- return true;
- if (tryNarrowMathIfNoOverflow(&I))
- return true;
+ if (!shouldKeepMulForMad64(I)) {
+ if (UseMul24Intrin && replaceMulWithMul24(I))
+ return true;
+ if (tryNarrowMathIfNoOverflow(&I))
+ return true;
+ }
bool Changed = false;
Instruction::BinaryOps Opc = I.getOpcode();
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24-mad64.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24-mad64.ll
new file mode 100644
index 0000000000000..f0ac959ea601d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24-mad64.ll
@@ -0,0 +1,84 @@
+; RUN: opt -S -mtriple=amdgpu8.03-- -passes=amdgpu-codegenprepare %s | FileCheck -check-prefixes=CHECK,GFX8 %s
+; RUN: opt -S -mtriple=amdgpu9.00-- -passes=amdgpu-codegenprepare %s | FileCheck -check-prefixes=CHECK,GFX9 %s
+
+; On GFX9+ a divergent i64 mul whose only user is an add stays a plain mul, so
+; the SDAG add combine can fold it into v_mad_[iu]64_[iu]32.
+
+define i64 @umul24_i64_add(i64 %lhs, i64 %rhs, i64 %acc) {
+; CHECK-LABEL: @umul24_i64_add(
+; GFX8: %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %{{.*}}, i32 %{{.*}})
+; GFX9-NOT: @llvm.amdgcn.mul
+; GFX9: %mul = mul i64 %lhs24, %rhs24
+; CHECK: %add = add i64 %mul, %acc
+ %lhs24 = and i64 %lhs, 16777215
+ %rhs24 = and i64 %rhs, 16777215
+ %mul = mul i64 %lhs24, %rhs24
+ %add = add i64 %mul, %acc
+ ret i64 %add
+}
+
+define i64 @smul24_i64_add(i64 %lhs, i64 %rhs, i64 %acc) {
+; CHECK-LABEL: @smul24_i64_add(
+; GFX8: %mul = call i64 @llvm.amdgcn.mul.i24.i64(i32 %{{.*}}, i32 %{{.*}})
+; GFX9-NOT: @llvm.amdgcn.mul
+; GFX9: %mul = mul i64 %lhs24, %rhs24
+; CHECK: %add = add i64 %mul, %acc
+ %lhs.shl = shl i64 %lhs, 40
+ %lhs24 = ashr i64 %lhs.shl, 40
+ %rhs.shl = shl i64 %rhs, 40
+ %rhs24 = ashr i64 %rhs.shl, 40
+ %mul = mul i64 %lhs24, %rhs24
+ %add = add i64 %mul, %acc
+ ret i64 %add
+}
+
+; A non-add user keeps the mul24 form on every target.
+define void @umul24_i64_store(i64 %lhs, i64 %rhs, ptr addrspace(1) %out) {
+; CHECK-LABEL: @umul24_i64_store(
+; CHECK: %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %{{.*}}, i32 %{{.*}})
+ %lhs24 = and i64 %lhs, 16777215
+ %rhs24 = and i64 %rhs, 16777215
+ %mul = mul i64 %lhs24, %rhs24
+ store i64 %mul, ptr addrspace(1) %out
+ ret void
+}
+
+; A second add user keeps the mul24 form on every target.
+define i64 @umul24_i64_two_adds(i64 %lhs, i64 %rhs, i64 %a, i64 %b) {
+; CHECK-LABEL: @umul24_i64_two_adds(
+; CHECK: %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %{{.*}}, i32 %{{.*}})
+ %lhs24 = and i64 %lhs, 16777215
+ %rhs24 = and i64 %rhs, 16777215
+ %mul = mul i64 %lhs24, %rhs24
+ %add0 = add i64 %mul, %a
+ %add1 = add i64 %mul, %b
+ %r = xor i64 %add0, %add1
+ ret i64 %r
+}
+
+; A divergent product that fits in 32 bits is not narrowed on GFX9 either.
+define i64 @umul16_i64_add(i64 %lhs, i64 %rhs, i64 %acc) {
+; CHECK-LABEL: @umul16_i64_add(
+; GFX8: %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %{{.*}}, i32 %{{.*}})
+; GFX9-NOT: mul i32
+; GFX9: %mul = mul i64 %lhs16, %rhs16
+; CHECK: %add = add i64 %mul, %acc
+ %lhs16 = and i64 %lhs, 65535
+ %rhs16 = and i64 %rhs, 65535
+ %mul = mul i64 %lhs16, %rhs16
+ %add = add i64 %mul, %acc
+ ret i64 %add
+}
+
+; Uniform values are still narrowed for the scalar unit.
+define amdgpu_kernel void @umul16_i64_add_uniform(ptr addrspace(1) %out, i64 %lhs, i64 %rhs, i64 %acc) {
+; CHECK-LABEL: @umul16_i64_add_uniform(
+; CHECK: [[MUL:%.*]] = mul i32 %{{.*}}, %{{.*}}
+; CHECK: zext i32 [[MUL]] to i64
+ %lhs16 = and i64 %lhs, 65535
+ %rhs16 = and i64 %rhs, 65535
+ %mul = mul i64 %lhs16, %rhs16
+ %add = add i64 %mul, %acc
+ store i64 %add, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 28e02c5434c6f..dcf6c7fdebae5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1960,7 +1960,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: v_readfirstlane_b32 s3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, s3
; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 5, v2, v[0:1]
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, v[0:1]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_nop 2
@@ -2000,7 +2000,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s3, v1
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 5, v2, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2038,7 +2038,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_readfirstlane_b32 s3, v1
; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, 5, v2, s[2:3]
+; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: s_mov_b32 s2, -1
; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2078,7 +2078,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-NEXT: v_readfirstlane_b32 s3, v1
; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
@@ -2117,7 +2117,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-NEXT: v_readfirstlane_b32 s3, v1
; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132-NEXT: s_mov_b32 s2, -1
; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
@@ -2156,7 +2156,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-NEXT: v_readfirstlane_b32 s3, v1
; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
@@ -2194,7 +2194,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-NEXT: v_readfirstlane_b32 s3, v1
; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1232-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1232-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
; GFX1232-NEXT: s_mov_b32 s2, -1
; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
@@ -2236,7 +2236,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-NEXT: v_readfirstlane_b32 s3, v1
; GFX1364-NEXT: v_readfirstlane_b32 s2, v0
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
@@ -2277,7 +2277,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-NEXT: v_readfirstlane_b32 s3, v1
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1332-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1332-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1332-NEXT: s_mov_b32 s3, 0x31016000
; GFX1332-NEXT: s_mov_b32 s2, -1
; GFX1332-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 4b16a4a371c18..e35004cd9ca59 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1863,7 +1863,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, s3
; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 5, v2, v[0:1]
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, v[0:1]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1898,7 +1898,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: v_readfirstlane_b32 s3, v1
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 5, v2, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -1931,7 +1931,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-NEXT: v_readfirstlane_b32 s3, v1
; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, 5, v2, s[2:3]
+; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: s_mov_b32 s2, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -1966,7 +1966,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1164-NEXT: v_readfirstlane_b32 s3, v1
; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -2000,7 +2000,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1132-NEXT: v_readfirstlane_b32 s3, v1
; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132-NEXT: s_mov_b32 s2, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -2035,7 +2035,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1364-NEXT: v_readfirstlane_b32 s3, v1
; GFX1364-NEXT: v_readfirstlane_b32 s2, v0
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
@@ -2069,7 +2069,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX1332-NEXT: v_readfirstlane_b32 s3, v1
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1332-NEXT: v_mad_co_u64_u32 v[0:1], null, 5, v2, s[2:3]
+; GFX1332-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]
; GFX1332-NEXT: s_mov_b32 s3, 0x31016000
; GFX1332-NEXT: s_mov_b32 s2, -1
; GFX1332-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
index 79017f3ea3cd7..143c21461dbb8 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
@@ -1175,51 +1175,45 @@ define i64 @dot4_acc64_not_dot4(i8 %a0, i8 %a1, i8 %b0, i8 %b1, i64 %acc) {
; GFX9-DL-LABEL: dot4_acc64_not_dot4:
; GFX9-DL: ; %bb.0:
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-DL-NEXT: v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_add_co_u32_e32 v1, vcc, v1, v4
-; GFX9-DL-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
-; GFX9-DL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-DL-NEXT: v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
+; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-DL-NEXT: v_mad_i64_i32 v[3:4], s[4:5], v1, v3, v[4:5]
+; GFX9-DL-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-DL-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[3:4]
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_acc64_not_dot4:
; GFX10-DL: ; %bb.0:
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_mul_hi_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_add_co_u32 v2, vcc_lo, v6, v4
-; GFX10-DL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo
-; GFX10-DL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-DL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v7, v1, vcc_lo
+; GFX10-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-DL-NEXT: v_mad_i64_i32 v[3:4], s4, v1, v3, v[4:5]
+; GFX10-DL-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX10-DL-NEXT: v_mad_i64_i32 v[0:1], s4, v0, v1, v[3:4]
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_acc64_not_dot4:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mul_hi_i32_i24_sdwa v9, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_mul_i32_i24_sdwa v8, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_mul_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[8:9], 0, v[4:5]
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[6:7], 0, v[0:1]
+; GFX950-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX950-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX950-NEXT: v_mad_i64_i32 v[4:5], s[0:1], v1, v3, v[4:5]
+; GFX950-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX950-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX950-NEXT: v_mad_i64_i32 v[0:1], s[0:1], v0, v1, v[4:5]
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_acc64_not_dot4:
; GFX9-NODL: ; %bb.0:
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_add_co_u32_e32 v1, vcc, v1, v4
-; GFX9-NODL-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
-; GFX9-NODL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-NODL-NEXT: v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
+; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-NODL-NEXT: v_mad_i64_i32 v[3:4], s[4:5], v1, v3, v[4:5]
+; GFX9-NODL-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-NODL-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[3:4]
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
%cv1e0 = sext i8 %a0 to i64
%cv1e1 = sext i8 %a1 to i64
diff --git a/llvm/test/CodeGen/AMDGPU/mad_u64_u32_mul24.ll b/llvm/test/CodeGen/AMDGPU/mad_u64_u32_mul24.ll
new file mode 100644
index 0000000000000..6eb8050346071
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mad_u64_u32_mul24.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.08 < %s | FileCheck -check-prefix=GCN %s
+
+; On GFX9+ AMDGPUCodeGenPrepare keeps a divergent i64 mul whose only user is an
+; add as a plain mul, so the add combine fuses it into v_mad_u64_u32 or
+; v_mad_i64_i32 instead of a mul24 + carry chain.
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+; Unsigned 24-bit product feeding an add.
+define amdgpu_kernel void @mul24_u_add_to_mad(ptr addrspace(1) %p, ptr addrspace(1) %q) {
+; GCN-LABEL: mul24_u_add_to_mad:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: global_load_dword v3, v2, s[0:1] offset:4
+; GCN-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GCN-NEXT: s_movk_i32 s2, 0xd1
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 8, v3
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v3, s2, v[0:1]
+; GCN-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pg = getelementptr i64, ptr addrspace(1) %p, i32 %tid
+ %qg = getelementptr i64, ptr addrspace(1) %q, i32 %tid
+ %x = load i64, ptr addrspace(1) %pg
+ %acc = load i64, ptr addrspace(1) %qg
+ %hi = lshr i64 %x, 40 ; <= 24 significant bits (u24)
+ %mul = mul i64 %hi, 209 ; 209 fits in 24 bits
+ %add = add i64 %mul, %acc
+ store i64 %add, ptr addrspace(1) %pg
+ ret void
+}
+
+; Signed 24-bit product feeding an add.
+define amdgpu_kernel void @mul24_i_add_to_mad(ptr addrspace(1) %p, ptr addrspace(1) %q) {
+; GCN-LABEL: mul24_i_add_to_mad:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: global_load_dword v3, v2, s[0:1] offset:4
+; GCN-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GCN-NEXT: s_movk_i32 s2, 0x64
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_ashrrev_i32_e32 v3, 8, v3
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mad_i64_i32 v[0:1], s[2:3], v3, s2, v[0:1]
+; GCN-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pg = getelementptr i64, ptr addrspace(1) %p, i32 %tid
+ %qg = getelementptr i64, ptr addrspace(1) %q, i32 %tid
+ %x = load i64, ptr addrspace(1) %pg
+ %acc = load i64, ptr addrspace(1) %qg
+ %sh = ashr i64 %x, 40 ; <= 24 significant signed bits (i24)
+ %mul = mul i64 %sh, 100 ; 100 fits in signed 24 bits
+ %add = add i64 %mul, %acc
+ store i64 %add, ptr addrspace(1) %pg
+ ret void
+}
+
+; A GEP user keeps the mul24 form. The ISel ptradd pattern forms the mad.
+define amdgpu_kernel void @mul24_u_ptradd_to_mad(ptr addrspace(1) %p) {
+; GCN-LABEL: mul24_u_ptradd_to_mad:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GCN-NEXT: s_movk_i32 s2, 0xd1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, s2, v[2:3]
+; GCN-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pg = getelementptr i64, ptr addrspace(1) %p, i32 %tid
+ %x = load i64, ptr addrspace(1) %pg
+ %hi = lshr i64 %x, 40
+ %off = mul i64 %hi, 209
+ %gep = getelementptr i8, ptr addrspace(1) %p, i64 %off
+ store i64 %x, ptr addrspace(1) %gep
+ ret void
+}
+
+; A second user keeps the mul24 form. The ISel pattern fuses the add and the
+; product is recomputed for the store.
+define amdgpu_kernel void @mul24_u_add_extra_use(ptr addrspace(1) %p, ptr addrspace(1) %q) {
+; GCN-LABEL: mul24_u_add_extra_use:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GCN-NEXT: s_movk_i32 s4, 0xd1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: global_load_dword v2, v4, s[0:1] offset:4
+; GCN-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3]
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 8, v2
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, s4, v[0:1]
+; GCN-NEXT: v_mul_hi_u32_u24_e32 v3, 0xd1, v2
+; GCN-NEXT: v_mul_u32_u24_e32 v2, 0xd1, v2
+; GCN-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GCN-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pg = getelementptr i64, ptr addrspace(1) %p, i32 %tid
+ %qg = getelementptr i64, ptr addrspace(1) %q, i32 %tid
+ %x = load i64, ptr addrspace(1) %pg
+ %acc = load i64, ptr addrspace(1) %qg
+ %hi = lshr i64 %x, 40
+ %mul = mul i64 %hi, 209
+ %add = add i64 %mul, %acc
+ store i64 %add, ptr addrspace(1) %pg
+ store i64 %mul, ptr addrspace(1) %qg ; second use of %mul
+ ret void
+}
More information about the llvm-commits
mailing list