[llvm-branch-commits] [llvm] [AMDGPU] Track GlobalISel known bits for mul24 intrinsics (PR #238437)
Alan Li via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 3 08:58:11 PST 2026
https://github.com/lialan created https://github.com/llvm/llvm-project/pull/238437
Depends on #237465.
Handle `amdgcn.mul.i24/u24` in GlobalISel known-bits analysis by sign- or zero-extending the low 24 operand bits before calling `KnownBits::mul`. This enables narrower shifts and simpler address calculations, including the negative-product bounds from the base PR.
Cover ignored high bits, zero operands and overflow. Include the affected shift/address checks and bf16-to-i128 check update.
Validation: rebuilt llc with assertions enabled for AMDGPU and AArch64; all five focused codegen tests and all 26 existing KnownBits tests passed. `git clang-format --staged` made no changes, and `git diff --check` passed. clang-tidy and Alive2 were unavailable.
The base branch mirrors the exact head of #237465. That PR originates from a fork, so GitHub native stacks are unavailable. Retarget this PR to main after #237465 merges.
>From dfb6b4cb18485d1a11203a995b6222ea7d9f70a4 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Tue, 3 Nov 2026 08:57:24 -0800
Subject: [PATCH] [AMDGPU] Track GlobalISel known bits for mul24 intrinsics
Sign or zero extend the low 24 operand bits before querying KnownBits::mul. This allows GlobalISel to narrow shifts and simplify address calculations through mul24 intrinsics.
Cover ignored high bits, zero operands and overflow, and update the affected codegen checks.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 18 ++
.../AMDGPU/GlobalISel/shl-ext-reduce.ll | 60 +++----
.../CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll | 165 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 2 -
llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll | 78 +++++++++
5 files changed, 286 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 349201d336628a..5b7b09cdab90cd 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20629,6 +20629,24 @@ void SITargetLowering::computeKnownBitsForTargetInstr(
case AMDGPU::G_INTRINSIC_CONVERGENT: {
Intrinsic::ID IID = cast<GIntrinsic>(MI)->getIntrinsicID();
switch (IID) {
+ case Intrinsic::amdgcn_mul_i24:
+ case Intrinsic::amdgcn_mul_u24: {
+ KnownBits Known2;
+ VT.computeKnownBitsImpl(MI->getOperand(2).getReg(), Known, DemandedElts,
+ Depth + 1);
+ VT.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts,
+ Depth + 1);
+ unsigned BitWidth = Known.getBitWidth();
+ if (IID == Intrinsic::amdgcn_mul_i24) {
+ Known = Known.trunc(24).sext(BitWidth);
+ Known2 = Known2.trunc(24).sext(BitWidth);
+ } else {
+ Known = Known.trunc(24).zext(BitWidth);
+ Known2 = Known2.trunc(24).zext(BitWidth);
+ }
+ Known = KnownBits::mul(Known, Known2);
+ break;
+ }
case Intrinsic::amdgcn_workitem_id_x:
knownBitsForWorkitemID(*getSubtarget(), VT, Known, 0);
break;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index 4aeb90aba22348..261e92848ca3c6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -230,12 +230,12 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX7-NEXT: v_and_b32_e32 v0, 6, v0
; GFX7-NEXT: v_mul_u32_u24_e32 v0, 7, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: v_lshl_b64 v[2:3], v[0:1], 2
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v1, v[2:3], s[0:3], 0 addr64
+; GFX7-NEXT: buffer_store_dword v1, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: mulu24_shl64:
@@ -243,14 +243,14 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX8-NEXT: v_and_b32_e32 v0, 6, v0
; GFX8-NEXT: v_mul_u32_u24_e32 v0, 7, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v4, v2
-; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GFX8-NEXT: flat_store_dword v[2:3], v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: mulu24_shl64:
@@ -258,14 +258,10 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_and_b32_e32 v0, 6, v0
; GFX9-NEXT: v_mul_u32_u24_e32 v0, 7, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
-; GFX9-NEXT: global_store_dword v[2:3], v1, off
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: mulu24_shl64:
@@ -274,13 +270,9 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX10-NEXT: v_and_b32_e32 v0, 6, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mul_u32_u24_e32 v0, 7, v0
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, s0
-; GFX10-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v5, v3, vcc_lo
-; GFX10-NEXT: global_store_dword v[2:3], v1, off
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: mulu24_shl64:
@@ -288,12 +280,9 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 6, v0
; GFX11-NEXT: v_mul_u32_u24_e32 v0, 7, v0
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
-; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, v5, v3, vcc_lo
-; GFX11-NEXT: global_store_b32 v[2:3], v1, off
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
bb:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -316,13 +305,14 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
; GFX7-NEXT: buffer_load_dword v1, v[1:2], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_mov_b64 s[2:3], s[6:7]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, 0xff800000, v1
-; GFX7-NEXT: v_mul_i32_i24_e32 v1, -7, v1
-; GFX7-NEXT: v_lshl_b64 v[3:4], v[1:2], 3
+; GFX7-NEXT: v_or_b32_e32 v0, 0xff800000, v1
+; GFX7-NEXT: v_mul_i32_i24_e32 v0, -7, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX7-NEXT: buffer_store_dwordx2 v[3:4], v[1:2], s[0:3], 0 addr64
+; GFX7-NEXT: buffer_store_dwordx2 v[1:2], v[3:4], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: muli24_shl64:
@@ -336,15 +326,15 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
; GFX8-NEXT: flat_load_dword v4, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v0, 0xff800000, v4
; GFX8-NEXT: v_mul_i32_i24_e32 v0, -7, v0
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -359,7 +349,7 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v1, 0xff800000, v1
; GFX9-NEXT: v_mul_i32_i24_e32 v1, -7, v1
-; GFX9-NEXT: v_lshlrev_b64 v[1:2], 3, v[1:2]
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX9-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -374,7 +364,7 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_e32 v1, 0xff800000, v1
; GFX10-NEXT: v_mul_i32_i24_e32 v1, -7, v1
-; GFX10-NEXT: v_lshlrev_b64 v[1:2], 3, v[1:2]
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX10-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -389,7 +379,7 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_or_b32_e32 v0, 0xff800000, v0
; GFX11-NEXT: v_mul_i32_i24_e32 v0, -7, v0
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
index 8ac2b293e49292..a2ea9a7b3eedbb 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -global-isel -global-isel-abort=1 -verify-machineinstrs < %s | FileCheck --check-prefix=GISEL %s
define weak_odr amdgpu_kernel void @test_mul24_knownbits_kernel(ptr addrspace(1) %p) #4 {
; GCN-LABEL: test_mul24_knownbits_kernel:
@@ -17,6 +18,23 @@ define weak_odr amdgpu_kernel void @test_mul24_knownbits_kernel(ptr addrspace(1)
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: global_store_dword v[0:1], v2, off
; GCN-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_mul24_knownbits_kernel:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GISEL-NEXT: v_and_b32_e32 v0, 3, v0
+; GISEL-NEXT: v_mul_i32_i24_e32 v0, -5, v0
+; GISEL-NEXT: v_and_b32_e32 v0, 0xffffffe0, v0
+; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GISEL-NEXT: s_endpgm
entry:
%0 = tail call i32 @llvm.amdgcn.workitem.id.x() #28, !range !4
%tid = and i32 %0, 3
@@ -37,6 +55,15 @@ define i32 @f(i32 %x, i32 %y) {
; GCN-NEXT: v_mul_i32_i24_e32 v0, v0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v0, 14, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: f:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v0, 0xffffff80, v0
+; GISEL-NEXT: v_or_b32_e32 v1, 0xffffff80, v1
+; GISEL-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 14, v0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%xx = or i32 %x, -128 ; 0xffffff80
%yy = or i32 %y, -128 ; 0xffffff80
%r = mul i32 %xx, %yy
@@ -48,3 +75,141 @@ define i32 @f(i32 %x, i32 %y) {
declare i32 @llvm.amdgcn.workitem.id.x() #20
!4 = !{i32 0, i32 1024}
+
+; Both operands are negative after sign extending their low 24 bits, even
+; though their i32 sign bits are unknown.
+define i32 @mul_i24_negative(i32 %x, i32 %y) {
+; GCN-LABEL: mul_i24_negative:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_i24_negative:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a = or i32 %x, 16777088
+ %b = or i32 %y, 16777088
+ %mul = call i32 @llvm.amdgcn.mul.i24(i32 %a, i32 %b)
+ %r = lshr i32 %mul, 15
+ ret i32 %r
+}
+
+; The positive operand is nonzero, so the product is strictly negative.
+define i32 @mul_i24_mixed(i32 %x, i32 %y) {
+; GCN-LABEL: mul_i24_mixed:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 0xfffff0, v0
+; GCN-NEXT: v_and_or_b32 v1, v1, 6, 1
+; GCN-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GCN-NEXT: v_ashrrev_i32_e32 v0, 7, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_i24_mixed:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v0, -1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a = or i32 %x, 16777200
+ %mask = and i32 %y, 7
+ %b = or i32 %mask, 1
+ %mul = call i32 @llvm.amdgcn.mul.i24(i32 %a, i32 %b)
+ %r = ashr i32 %mul, 7
+ ret i32 %r
+}
+
+define i32 @mul_i24_may_be_zero(i32 %x, i32 %y) {
+; GCN-LABEL: mul_i24_may_be_zero:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 0xfffff0, v0
+; GCN-NEXT: v_and_b32_e32 v1, 7, v1
+; GCN-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GCN-NEXT: v_ashrrev_i32_e32 v0, 7, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_i24_may_be_zero:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v0, 0xfffff0, v0
+; GISEL-NEXT: v_and_b32_e32 v1, 7, v1
+; GISEL-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GISEL-NEXT: v_ashrrev_i32_e32 v0, 7, v0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a = or i32 %x, 16777200
+ %b = and i32 %y, 7
+ %mul = call i32 @llvm.amdgcn.mul.i24(i32 %a, i32 %b)
+ %r = ashr i32 %mul, 7
+ ret i32 %r
+}
+
+; A product of two negative i24 values can overflow i32.
+define i32 @mul_i24_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: mul_i24_overflow:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 0x800000, v0
+; GCN-NEXT: v_or_b32_e32 v1, 0x800000, v1
+; GCN-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 31, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_i24_overflow:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v0, 0x800000, v0
+; GISEL-NEXT: v_or_b32_e32 v1, 0x800000, v1
+; GISEL-NEXT: v_mul_i32_i24_e32 v0, v0, v1
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a = or i32 %x, 8388608
+ %b = or i32 %y, 8388608
+ %mul = call i32 @llvm.amdgcn.mul.i24(i32 %a, i32 %b)
+ %r = lshr i32 %mul, 31
+ ret i32 %r
+}
+
+; Bits 24-31 are ignored, including the i32 sign bit.
+define i32 @mul_u24_high_bits(i32 %x, i32 %y) {
+; GCN-LABEL: mul_u24_high_bits:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_u24_high_bits:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %mask = and i32 %x, -16777209
+ %a = or i32 %mask, -2147483648
+ %mul = call i32 @llvm.amdgcn.mul.u24(i32 %a, i32 %y)
+ %r = lshr i32 %mul, 27
+ ret i32 %r
+}
+
+define i32 @mul_u24_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: mul_u24_overflow:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 31, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: mul_u24_overflow:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v0
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ %mul = call i32 @llvm.amdgcn.mul.u24(i32 %x, i32 %y)
+ %r = lshr i32 %mul, 31
+ ret i32 %r
+}
+
+declare i32 @llvm.amdgcn.mul.i24(i32, i32)
+declare i32 @llvm.amdgcn.mul.u24(i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index bd61e4a1ee2be4..30cc06c4099240 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -910,9 +910,7 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
; GISEL-NEXT: v_sub_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GISEL-NEXT: v_lshrrev_b16_e32 v0, v0, v4
; GISEL-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v7
-; GISEL-NEXT: v_ashrrev_i32_e32 v2, 31, v1
; GISEL-NEXT: v_mul_i32_i24_e32 v0, v0, v7
-; GISEL-NEXT: v_mov_b32_e32 v3, v2
; GISEL-NEXT: ; %bb.5: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GISEL-NEXT: .LBB6_6: ; %fp-to-i-cleanup
diff --git a/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll b/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
index 4c08709b8aedb8..c1d734085a185a 100644
--- a/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu6.01-- < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.00-- < %s | FileCheck -check-prefix=GFX9 %s
define amdgpu_kernel void @zext_shl64_to_32(ptr addrspace(1) nocapture %out, i32 %x) {
; GCN-LABEL: zext_shl64_to_32:
@@ -14,6 +15,17 @@ define amdgpu_kernel void @zext_shl64_to_32(ptr addrspace(1) nocapture %out, i32
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: zext_shl64_to_32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_lshl_b32 s2, s2, 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%and = and i32 %x, 1073741823
%ext = zext i32 %and to i64
%shl = shl i64 %ext, 2
@@ -35,6 +47,18 @@ define amdgpu_kernel void @sext_shl64_to_32(ptr addrspace(1) nocapture %out, i32
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: sext_shl64_to_32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s2, s2, 0x1fffffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%and = and i32 %x, 536870911
%ext = sext i32 %and to i64
%shl = shl i64 %ext, 2
@@ -57,6 +81,20 @@ define amdgpu_kernel void @zext_shl64_overflow(ptr addrspace(1) nocapture %out,
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: zext_shl64_overflow:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bitset0_b32 s2, 31
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%and = and i32 %x, 2147483647
%ext = zext i32 %and to i64
%shl = shl i64 %ext, 2
@@ -79,6 +117,20 @@ define amdgpu_kernel void @sext_shl64_overflow(ptr addrspace(1) nocapture %out,
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: sext_shl64_overflow:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bitset0_b32 s2, 31
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%and = and i32 %x, 2147483647
%ext = sext i32 %and to i64
%shl = shl i64 %ext, 2
@@ -99,6 +151,17 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_store_dword v1, v[0:1], s[0:3], 0 addr64
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: mulu24_shl64:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_and_b32_e32 v0, 6, v0
+; GFX9-NEXT: v_mul_u32_u24_e32 v0, 7, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
bb:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%tmp1 = and i32 %tmp, 6
@@ -129,6 +192,21 @@ define amdgpu_kernel void @muli24_shl64(ptr addrspace(1) nocapture %arg, ptr add
; GCN-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GCN-NEXT: buffer_store_dwordx2 v[1:2], v[3:4], s[0:3], 0 addr64
; GCN-NEXT: s_endpgm
+;
+; GFX9-LABEL: muli24_shl64:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_e32 v1, 0x800000, v1
+; GFX9-NEXT: v_mul_i32_i24_e32 v1, -7, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX9-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
bb:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%tmp2 = sext i32 %tmp to i64
More information about the llvm-branch-commits
mailing list