[llvm] b966c71 - [AMDGPU] Fix true16 high-half packing when the low operand is not a VGPR (#220843)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 00:49:14 PDT 2026
Author: Barbara Mitic
Date: 2026-09-07T08:49:09+01:00
New Revision: b966c71b74e498c3a9570c790a71a5a1a2fe0e2f
URL: https://github.com/llvm/llvm-project/commit/b966c71b74e498c3a9570c790a71a5a1a2fe0e2f
DIFF: https://github.com/llvm/llvm-project/commit/b966c71b74e498c3a9570c790a71a5a1a2fe0e2f.diff
LOG: [AMDGPU] Fix true16 high-half packing when the low operand is not a VGPR (#220843)
The true16 (hi << 16) | z pattern used v_or_b16 and EXTRACT_SUBREG hi16
on z. That is only valid when z is a VGPR. When z is an immediate (e.g.
0x3c00), EXTRACT_SUBREG hi16 does not isolate the high 16 bits, so the
whole constant is ORed into hi: we get (hi | 0x3c00) << 16 instead of hi
<< 16.
Select v_lshl_or_b32 instead, with z as VSrc_b32, matching the fake16
path.
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstructions.td
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
llvm/test/CodeGen/AMDGPU/pack-half-to-lshl-or.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 554c2ba51bab8..af08bf0861215 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -4094,17 +4094,16 @@ def : GCNPat <
(V_LSHL_OR_B32_e64 VGPR_32:$src_hi, (i32 16), VSrc_b32:$src2)
>;
-// Real-true16: high source is a VGPR_16, so OR it into the high half of src2
-// with v_or_b16 and assemble the 32-bit result with a REG_SEQUENCE.
+// Real-true16: high source is a VGPR_16. Widen it to a 32-bit register and
+// fold into v_lshl_or_b32.
let True16Predicate = UseRealTrue16Insts in
def : GCNPat <
(i32 (DivergentBinFrag<or>
(i32 (bitconvert (v2i16 (build_vector (i16 0), (i16 VGPR_16:$src_hi))))),
i32:$src2)),
- (REG_SEQUENCE VGPR_32,
- (i16 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS $src2, VGPR_32)), lo16)), lo16,
- (V_OR_B16_t16_e64 (i32 0), VGPR_16:$src_hi,
- (i32 0), (i16 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS $src2, VGPR_32)), hi16))), hi16)
+ (V_LSHL_OR_B32_e64
+ (i32 (REG_SEQUENCE VGPR_32, VGPR_16:$src_hi, lo16, (i16 (IMPLICIT_DEF)), hi16)),
+ (i32 16), VSrc_b32:$src2)
>;
// With multiple uses of the shift, this will duplicate the shift and
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index 6df159cb9cbef..ec16890ba93fb 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -59,8 +59,8 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-REAL16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
; GCN-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GCN-SDAG-REAL16-NEXT: v_or_b16 v0.l, v6.l, v0.l
-; GCN-SDAG-REAL16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v1, off
+; GCN-SDAG-REAL16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v0, off
; GCN-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
%a = load i8, ptr addrspace(1) %ptr_a
%b = load i8, ptr addrspace(1) %ptr_b
diff --git a/llvm/test/CodeGen/AMDGPU/pack-half-to-lshl-or.ll b/llvm/test/CodeGen/AMDGPU/pack-half-to-lshl-or.ll
index a57a42f56c117..2918684bef7ba 100644
--- a/llvm/test/CodeGen/AMDGPU/pack-half-to-lshl-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack-half-to-lshl-or.ll
@@ -29,9 +29,7 @@ define i32 @pack_high_half(i16 %hi, i32 %lo) {
; GFX11-TRUE16-LABEL: pack_high_half:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: pack_high_half:
@@ -51,9 +49,7 @@ define i32 @pack_high_half(i16 %hi, i32 %lo) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
%ext = zext i16 %hi to i32
%shl = shl i32 %ext, 16
@@ -89,8 +85,8 @@ define void @pack_high_half_to_store(i16 %hi, i32 %lo, ptr addrspace(1) %out) {
; GFX11-TRUE16-LABEL: pack_high_half_to_store:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX11-TRUE16-NEXT: global_store_b32 v[2:3], v1, off
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX11-TRUE16-NEXT: global_store_b32 v[2:3], v0, off
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: pack_high_half_to_store:
@@ -111,8 +107,8 @@ define void @pack_high_half_to_store(i16 %hi, i32 %lo, ptr addrspace(1) %out) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX12-TRUE16-NEXT: global_store_b32 v[2:3], v1, off
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX12-TRUE16-NEXT: global_store_b32 v[2:3], v0, off
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
%ext = zext i16 %hi to i32
%shl = shl i32 %ext, 16
@@ -144,10 +140,7 @@ define i32 @pack_high_half_sgpr(i16 %hi, i32 inreg %lo) {
; GFX11-TRUE16-LABEL: pack_high_half_sgpr:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: pack_high_half_sgpr:
@@ -167,10 +160,7 @@ define i32 @pack_high_half_sgpr(i16 %hi, i32 inreg %lo) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
%ext = zext i16 %hi to i32
%shl = shl i32 %ext, 16
@@ -201,10 +191,7 @@ define i32 @pack_high_half_sgpr.half(half %hi, i32 inreg %lo) {
; GFX11-TRUE16-LABEL: pack_high_half_sgpr.half:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: pack_high_half_sgpr.half:
@@ -224,10 +211,7 @@ define i32 @pack_high_half_sgpr.half(half %hi, i32 inreg %lo) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
%bc = bitcast half %hi to i16
%ext = zext i16 %bc to i32
@@ -259,10 +243,7 @@ define i32 @pack_high_half_sgpr.bfloat(bfloat %hi, i32 inreg %lo) {
; GFX11-TRUE16-LABEL: pack_high_half_sgpr.bfloat:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: pack_high_half_sgpr.bfloat:
@@ -282,10 +263,7 @@ define i32 @pack_high_half_sgpr.bfloat(bfloat %hi, i32 inreg %lo) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
%bc = bitcast bfloat %hi to i16
%ext = zext i16 %bc to i32
@@ -293,3 +271,116 @@ define i32 @pack_high_half_sgpr.bfloat(bfloat %hi, i32 inreg %lo) {
%or = or i32 %shl, %lo
ret i32 %or
}
+
+define i32 @pack_high_half_imm(i16 %hi) {
+; GFX8-LABEL: pack_high_half_imm:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: pack_high_half_imm:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_movk_i32 s4, 0x3c00
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, s4
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: pack_high_half_imm:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: pack_high_half_imm:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: pack_high_half_imm:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: pack_high_half_imm:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+ %ext = zext i16 %hi to i32
+ %shl = shl i32 %ext, 16
+ %or = or i32 %shl, 15360
+ ret i32 %or
+}
+
+define void @pack_high_half_imm_to_store(i16 %hi, ptr addrspace(1) %out) {
+; GFX8-LABEL: pack_high_half_imm_to_store:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; GFX8-NEXT: flat_store_dword v[1:2], v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: pack_high_half_imm_to_store:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_movk_i32 s4, 0x3c00
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, s4
+; GFX9-NEXT: global_store_dword v[1:2], v0, off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: pack_high_half_imm_to_store:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX11-FAKE16-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: pack_high_half_imm_to_store:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX11-TRUE16-NEXT: global_store_b32 v[1:2], v0, off
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: pack_high_half_imm_to_store:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX12-FAKE16-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: pack_high_half_imm_to_store:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0x3c00
+; GFX12-TRUE16-NEXT: global_store_b32 v[1:2], v0, off
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+ %ext = zext i16 %hi to i32
+ %shl = shl i32 %ext, 16
+ %or = or i32 %shl, 15360
+ store i32 %or, ptr addrspace(1) %out
+ ret void
+}
More information about the llvm-commits
mailing list