[llvm] [AMDGPU] Add pattern to select scalar ops for fshr with uniform operands (PR #165295)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 4 11:03:26 PST 2025
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/165295
>From f03b6f97dc0855d4a2606a712d94ff0c6f777fbc Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Mon, 27 Oct 2025 13:49:05 -0500
Subject: [PATCH 1/5] add ISEL pattern for fshr with uniform operands
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 5 +
llvm/test/CodeGen/AMDGPU/fshr.ll | 518 +++++++++++++++++------
2 files changed, 393 insertions(+), 130 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 6f1feb1dc2996..6e64a1514d27d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2763,6 +2763,11 @@ def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
/* src2_modifiers */ 0,
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
+
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
+ (S_OR_B32 (S_LSHR_B32 $src1, (S_AND_B32 $src2, (i32 0xffffffff))), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), (S_AND_B32 $src2, (i32 0xffffffff)))))
+>;
+
} // end True16Predicate = UseFakeTrue16Insts
/********** ====================== **********/
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index ef68f44bac203..4be928f0ef6f1 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -103,10 +103,15 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, -1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
;
; GFX12-TRUE16-LABEL: fshr_i32:
@@ -128,10 +133,15 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX12-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, v0
-; GFX12-FAKE16-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, -1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
+; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
@@ -195,23 +205,49 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: fshr_i32_imm:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, s2, s3, 7
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: fshr_i32_imm:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_alignbit_b32 v1, s2, s3, 7
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: fshr_i32_imm:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s2, s3, 7
+; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: fshr_i32_imm:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, 32, 7
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: fshr_i32_imm:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s2, s3, 7
+; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: fshr_i32_imm:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s4, 32, 7
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s2, s4
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, 7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 7)
store i32 %0, ptr addrspace(1) %in
@@ -321,12 +357,20 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s7
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s1, s3, v0
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s2, v2
-; GFX11-FAKE16-NEXT: global_store_b64 v3, v[0:1], s[4:5]
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, -1
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, -1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-FAKE16-NEXT: s_sub_i32 s7, 32, s7
+; GFX11-FAKE16-NEXT: s_sub_i32 s8, 32, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
;
; GFX12-TRUE16-LABEL: fshr_v2i32:
@@ -352,12 +396,20 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s7
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s1, s3, v0
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, s2, v2
-; GFX12-FAKE16-NEXT: global_store_b64 v3, v[0:1], s[4:5]
+; GFX12-FAKE16-NEXT: s_and_b32 s7, s7, -1
+; GFX12-FAKE16-NEXT: s_and_b32 s6, s6, -1
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s7, 32, s7
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s8, 32, s6
+; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
+; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
+; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
@@ -433,29 +485,69 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: fshr_v2i32_imm:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, s1, s3, 9
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s2, 7
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: fshr_v2i32_imm:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_alignbit_b32 v1, s1, s3, 9
-; GFX12-NEXT: v_alignbit_b32 v0, s0, s2, 7
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX12-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: fshr_v2i32_imm:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, 9
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, 7
+; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: fshr_v2i32_imm:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_sub_i32 s6, 32, 9
+; GFX11-FAKE16-NEXT: s_sub_i32 s7, 32, 7
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s6
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 9
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: fshr_v2i32_imm:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, 9
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, 7
+; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: fshr_v2i32_imm:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s6, 32, 9
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s7, 32, 7
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, s6
+; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s2, 7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, 9
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
store <2 x i32> %0, ptr addrspace(1) %in
@@ -595,17 +687,32 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v3, s11, s15, v0
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v2, s10, s14, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s9, s13, v4
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s8, s12, v5
-; GFX11-FAKE16-NEXT: global_store_b128 v6, v[0:3], s[4:5]
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, -1
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, -1
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, -1
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, -1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
+; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
+; GFX11-FAKE16-NEXT: s_sub_i32 s2, 32, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s13, s1
+; GFX11-FAKE16-NEXT: s_sub_i32 s1, 32, s1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s12, s0
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, 32, s0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s11, s3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s10, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s9, s1
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s8, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s6, s3
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s7, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s12, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s13, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
;
; GFX12-TRUE16-LABEL: fshr_v4i32:
@@ -635,17 +742,32 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v3, s11, s15, v0
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v2, s10, s14, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s9, s13, v4
-; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s8, s12, v5
-; GFX12-FAKE16-NEXT: global_store_b128 v6, v[0:3], s[4:5]
+; GFX12-FAKE16-NEXT: s_and_b32 s3, s3, -1
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, -1
+; GFX12-FAKE16-NEXT: s_and_b32 s1, s1, -1
+; GFX12-FAKE16-NEXT: s_and_b32 s0, s0, -1
+; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s2, 32, s2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s13, s13, s1
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s1, 32, s1
+; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s12, s0
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s0, 32, s0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s11, s3
+; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s10, s2
+; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s9, s1
+; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s8, s0
+; GFX12-FAKE16-NEXT: s_or_b32 s3, s6, s3
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s7, s2
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s12, s0
+; GFX12-FAKE16-NEXT: s_or_b32 s1, s13, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z)
@@ -737,33 +859,89 @@ define amdgpu_kernel void @fshr_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: fshr_v4i32_imm:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v3, s11, s15, 1
-; GFX11-NEXT: v_alignbit_b32 v2, s10, s14, 9
-; GFX11-NEXT: v_alignbit_b32 v1, s9, s13, 7
-; GFX11-NEXT: v_alignbit_b32 v0, s8, s12, 1
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: fshr_v4i32_imm:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_alignbit_b32 v3, s11, s15, 1
-; GFX12-NEXT: v_alignbit_b32 v2, s10, s14, 9
-; GFX12-NEXT: v_alignbit_b32 v1, s9, s13, 7
-; GFX12-NEXT: v_alignbit_b32 v0, s8, s12, 1
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: fshr_v4i32_imm:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, 1
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, 9
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, 7
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, 1
+; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: fshr_v4i32_imm:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_sub_i32 s2, 32, 1
+; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, 9
+; GFX11-FAKE16-NEXT: s_sub_i32 s6, 32, 7
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s11, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s15, 1
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s10, s3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s14, 9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s9, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s13, 7
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s8, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s12, 1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s7, s3
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s8, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s9, s6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: fshr_v4i32_imm:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, 1
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, 9
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, 7
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, 1
+; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: fshr_v4i32_imm:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s2, 32, 1
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, 9
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s6, 32, 7
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s11, s2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s5, s15, 1
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s10, s3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s7, s14, 9
+; GFX12-FAKE16-NEXT: s_lshl_b32 s6, s9, s6
+; GFX12-FAKE16-NEXT: s_lshr_b32 s9, s13, 7
+; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s8, s2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s12, 1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX12-FAKE16-NEXT: s_or_b32 s3, s7, s3
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s8, s2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, s9, s6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
entry:
%0 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 1, i32 7, i32 9, i32 33>)
store <4 x i32> %0, ptr addrspace(1) %in
@@ -2091,29 +2269,109 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %src0, <2 x i24> %src1, <2 x i24> %src2
; GFX10-NEXT: v_alignbit_b32 v1, v1, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_mul_hi_u32 v6, 0xaaaaaab, v4
-; GFX11-NEXT: v_mul_hi_u32 v7, 0xaaaaaab, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_u32_u24_e32 v6, 24, v6
-; GFX11-NEXT: v_mul_u32_u24_e32 v7, 24, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v4, v4, v6
-; GFX11-NEXT: v_sub_nc_u32_e32 v5, v5, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 8, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 8, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v2, v4
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshr_v2i24:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mul_hi_u32 v6, 0xaaaaaab, v4
+; GFX11-TRUE16-NEXT: v_mul_hi_u32 v7, 0xaaaaaab, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v6, 24, v6
+; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v7, 24, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v4, v4, v6
+; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v5, v5, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 8, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_fshr_v2i24:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_mul_hi_u32 v6, 0xaaaaaab, v4
+; GFX11-FAKE16-NEXT: v_mul_hi_u32 v7, 0xaaaaaab, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v6, 24, v6
+; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v7, 24, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v4, v4, v6
+; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 8, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 8, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v1, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: v_fshr_v2i24:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mul_hi_u32 v6, 0xaaaaaab, v4
+; GFX12-TRUE16-NEXT: v_mul_hi_u32 v7, 0xaaaaaab, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mul_u32_u24_e32 v6, 24, v6
+; GFX12-TRUE16-NEXT: v_mul_u32_u24_e32 v7, 24, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_nc_u32_e32 v4, v4, v6
+; GFX12-TRUE16-NEXT: v_sub_nc_u32_e32 v5, v5, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 8, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v5, 8, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: v_fshr_v2i24:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_mul_hi_u32 v6, 0xaaaaaab, v4
+; GFX12-FAKE16-NEXT: v_mul_hi_u32 v7, 0xaaaaaab, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_mul_u32_u24_e32 v6, 24, v6
+; GFX12-FAKE16-NEXT: v_mul_u32_u24_e32 v7, 24, v7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_nc_u32_e32 v4, v4, v6
+; GFX12-FAKE16-NEXT: v_sub_nc_u32_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 8, v4
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v5, 8, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v1, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%ret = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %src0, <2 x i24> %src1, <2 x i24> %src2)
ret <2 x i24> %ret
}
>From adf0f1948ef3d1c5e2f5eba46e2f0d74d8e28b23 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 28 Oct 2025 12:06:58 -0500
Subject: [PATCH 2/5] remove redundant S_ADD nop
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 +-
llvm/test/CodeGen/AMDGPU/fshr.ll | 24 ++++--------------------
2 files changed, 5 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 6e64a1514d27d..bb02a562aa0da 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2765,7 +2765,7 @@ def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
>;
def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
- (S_OR_B32 (S_LSHR_B32 $src1, (S_AND_B32 $src2, (i32 0xffffffff))), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), (S_AND_B32 $src2, (i32 0xffffffff)))))
+ (S_OR_B32 (S_LSHR_B32 $src1, $src2), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), $src2)))
>;
} // end True16Predicate = UseFakeTrue16Insts
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 4be928f0ef6f1..59bda3df8c76e 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -103,13 +103,11 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, -1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s2
; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -133,13 +131,11 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX12-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, -1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s2
; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
@@ -357,11 +353,9 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, -1
-; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, -1
+; GFX11-FAKE16-NEXT: s_sub_i32 s8, 32, s6
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
; GFX11-FAKE16-NEXT: s_sub_i32 s7, 32, s7
-; GFX11-FAKE16-NEXT: s_sub_i32 s8, 32, s6
; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
@@ -396,11 +390,9 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_and_b32 s7, s7, -1
-; GFX12-FAKE16-NEXT: s_and_b32 s6, s6, -1
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s8, 32, s6
; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
; GFX12-FAKE16-NEXT: s_sub_co_i32 s7, 32, s7
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s8, 32, s6
; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
@@ -688,10 +680,6 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, -1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, -1
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, -1
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, -1
; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s3
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
@@ -743,10 +731,6 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_and_b32 s3, s3, -1
-; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, -1
-; GFX12-FAKE16-NEXT: s_and_b32 s1, s1, -1
-; GFX12-FAKE16-NEXT: s_and_b32 s0, s0, -1
; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s3
; GFX12-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
>From a46e1087e4d6e5bb61d2ce8cf19f44acce35de8e Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Wed, 29 Oct 2025 17:31:35 -0500
Subject: [PATCH 3/5] replace previous instr sequence with reg_sequence and
s_lshr_b64 based one
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 12 +-
llvm/test/CodeGen/AMDGPU/fshr.ll | 236 ++++++++++-------------
2 files changed, 117 insertions(+), 131 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index bb02a562aa0da..e7e0dd72e78dd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2764,10 +2764,20 @@ def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
+// The commented out code has been left intentionally to aid the review process, if needed.
+// Will delete before landing.
+//def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
+// (S_OR_B32 (S_LSHR_B32 $src1, $src2), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), $src2)))
+//>;
+
def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
- (S_OR_B32 (S_LSHR_B32 $src1, $src2), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), $src2)))
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
>;
+//def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
+// (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
+//>;
+
} // end True16Predicate = UseFakeTrue16Insts
/********** ====================== **********/
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 59bda3df8c76e..4fccebabce04f 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -103,11 +103,11 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s2, 31
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -131,11 +131,11 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX12-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, s2
-; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s0
+; GFX12-FAKE16-NEXT: s_and_b32 s0, s2, 31
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
@@ -213,12 +213,11 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX11-FAKE16-LABEL: fshr_i32_imm:
; GFX11-FAKE16: ; %bb.0: ; %entry
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_sub_i32 s4, 32, 7
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, s4
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 7
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -235,12 +234,11 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX12-FAKE16-LABEL: fshr_i32_imm:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s4, 32, 7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s2, s4
-; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, 7
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
@@ -349,20 +347,19 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX11-FAKE16-LABEL: fshr_v2i32:
; GFX11-FAKE16: ; %bb.0: ; %entry
; GFX11-FAKE16-NEXT: s_clause 0x2
-; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_sub_i32 s8, 32, s6
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-FAKE16-NEXT: s_sub_i32 s7, 32, s7
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s6, 31
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 31
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -386,20 +383,19 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX12-FAKE16-LABEL: fshr_v2i32:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_clause 0x2
-; GFX12-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s8, 32, s6
-; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s7, 32, s7
-; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
-; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
-; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s2, s0
-; GFX12-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s0
+; GFX12-FAKE16-NEXT: s_and_b32 s0, s6, 31
+; GFX12-FAKE16-NEXT: s_and_b32 s6, s7, 31
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
@@ -494,17 +490,14 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_sub_i32 s6, 32, 9
-; GFX11-FAKE16-NEXT: s_sub_i32 s7, 32, 7
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s6
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s7
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 7
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 9
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s1
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -526,17 +519,14 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s6, 32, 9
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s7, 32, 7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s1, s6
-; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s0, s7
-; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s2, 7
-; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s3, 9
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s2, s0
-; GFX12-FAKE16-NEXT: s_or_b32 s1, s3, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s0
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s1
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
@@ -676,30 +666,28 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX11-FAKE16-LABEL: fshr_v4i32:
; GFX11-FAKE16: ; %bb.0: ; %entry
; GFX11-FAKE16-NEXT: s_clause 0x2
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
-; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, s3
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
-; GFX11-FAKE16-NEXT: s_sub_i32 s2, 32, s2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s13, s1
-; GFX11-FAKE16-NEXT: s_sub_i32 s1, 32, s1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s12, s0
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, 32, s0
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s11, s3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s10, s2
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s9, s1
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s8, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s3, s6, s3
-; GFX11-FAKE16-NEXT: s_or_b32 s2, s7, s2
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s12, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s13, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s15
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT: s_and_b32 s11, s3, 31
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s10
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s2, 31
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s9
+; GFX11-FAKE16-NEXT: s_and_b32 s16, s1, 31
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s0, 31
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX11-FAKE16-NEXT: s_endpgm
;
@@ -727,30 +715,28 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX12-FAKE16-LABEL: fshr_v4i32:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_clause 0x2
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s15, s3
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, s3
-; GFX12-FAKE16-NEXT: s_lshr_b32 s7, s14, s2
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s2, 32, s2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s13, s13, s1
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s1, 32, s1
-; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s12, s0
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s0, 32, s0
-; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s11, s3
-; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s10, s2
-; GFX12-FAKE16-NEXT: s_lshl_b32 s1, s9, s1
-; GFX12-FAKE16-NEXT: s_lshl_b32 s0, s8, s0
-; GFX12-FAKE16-NEXT: s_or_b32 s3, s6, s3
-; GFX12-FAKE16-NEXT: s_or_b32 s2, s7, s2
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s12, s0
-; GFX12-FAKE16-NEXT: s_or_b32 s1, s13, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s15
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s11
+; GFX12-FAKE16-NEXT: s_and_b32 s11, s3, 31
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s10
+; GFX12-FAKE16-NEXT: s_and_b32 s10, s2, 31
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s9
+; GFX12-FAKE16-NEXT: s_and_b32 s16, s1, 31
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s8
+; GFX12-FAKE16-NEXT: s_and_b32 s8, s0, 31
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s6
; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
@@ -862,26 +848,21 @@ define amdgpu_kernel void @fshr_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_sub_i32 s2, 32, 1
-; GFX11-FAKE16-NEXT: s_sub_i32 s3, 32, 9
-; GFX11-FAKE16-NEXT: s_sub_i32 s6, 32, 7
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s11, s2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s15, 1
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s10, s3
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s14, 9
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s9, s6
-; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s13, 7
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s8, s2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s12, 1
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX11-FAKE16-NEXT: s_or_b32 s3, s7, s3
-; GFX11-FAKE16-NEXT: s_or_b32 s2, s8, s2
-; GFX11-FAKE16-NEXT: s_or_b32 s5, s9, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s15
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s8
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
;
@@ -904,26 +885,21 @@ define amdgpu_kernel void @fshr_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s2, 32, 1
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s3, 32, 9
-; GFX12-FAKE16-NEXT: s_sub_co_i32 s6, 32, 7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s11, s2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s5, s15, 1
-; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s10, s3
-; GFX12-FAKE16-NEXT: s_lshr_b32 s7, s14, 9
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, s9, s6
-; GFX12-FAKE16-NEXT: s_lshr_b32 s9, s13, 7
-; GFX12-FAKE16-NEXT: s_lshl_b32 s2, s8, s2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s12, 1
-; GFX12-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX12-FAKE16-NEXT: s_or_b32 s3, s7, s3
-; GFX12-FAKE16-NEXT: s_or_b32 s2, s8, s2
-; GFX12-FAKE16-NEXT: s_or_b32 s5, s9, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s15
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s8
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX12-FAKE16-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s6
; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
>From a862618f6c8dba05e5db15d438dc5bcce062df77 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Sat, 1 Nov 2025 16:58:52 -0500
Subject: [PATCH 4/5] extending new uniform patterns to all fshr cases;tests
updated
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 36 +-
llvm/lib/Target/AMDGPU/SOPInstructions.td | 11 +
llvm/test/CodeGen/AMDGPU/fshl.ll | 380 ++++---
llvm/test/CodeGen/AMDGPU/fshr.ll | 1191 +++++++++++++--------
4 files changed, 1022 insertions(+), 596 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e7e0dd72e78dd..f71f7ea42d48e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2694,12 +2694,21 @@ def : GCNPat<pat,
$src1, /* clamp */ 0, /* op_sel */ 0)
>;
-def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
+def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
(V_ALIGNBIT_B32_opsel_e64 /* src0_modifiers */ 0, $src0,
/* src1_modifiers */ 0, $src1,
/* src2_modifiers */ 0,
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
+
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
+>;
+
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
+>;
+
} // isGFX9GFX10
} // end True16Predicate = NotHasTrue16BitInsts
@@ -2722,12 +2731,21 @@ def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:
(i16 (EXTRACT_SUBREG VGPR_32:$src1, lo16)),
/* clamp */ 0, /* op_sel */ 0)>;
-def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
+def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
(V_ALIGNBIT_B32_t16_e64 /* src0_modifiers */ 0, $src0,
/* src1_modifiers */ 0, $src1,
/* src2_modifiers */ 0,
(EXTRACT_SUBREG VGPR_32:$src2, lo16),
/* clamp */ 0, /* op_sel */ 0)>;
+
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
+>;
+
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
+>;
+
} // end True16Predicate = UseRealTrue16Insts
let True16Predicate = UseFakeTrue16Insts in {
@@ -2757,26 +2775,20 @@ def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:
$src1, /* clamp */ 0, /* op_sel */ 0)
>;
-def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
+def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
(V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,
/* src1_modifiers */ 0, $src1,
/* src2_modifiers */ 0,
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
-// The commented out code has been left intentionally to aid the review process, if needed.
-// Will delete before landing.
-//def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
-// (S_OR_B32 (S_LSHR_B32 $src1, $src2), (S_LSHL_B32 $src0, (S_SUB_I32 (i32 32), $src2)))
-//>;
-
def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
(i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
>;
-//def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
-// (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
-//>;
+def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
+>;
} // end True16Predicate = UseFakeTrue16Insts
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 84287b621fe78..bc54ef847305d 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -207,6 +207,17 @@ class DivergentBinFrag<SDPatternOperator Op> : PatFrag <
let GISelPredicateCode = [{return true;}];
}
+class DivergentTernaryFrag<SDPatternOperator Op> : PatFrag <
+ (ops node:$src0, node:$src1, node:$src2),
+ (Op $src0, $src1, $src2),
+ [{ return N->isDivergent(); }]> {
+ // This check is unnecessary as it's captured by the result register
+ // bank constraint.
+ //
+ // FIXME: Should add a way for the emitter to recognize this is a
+ // trivially true predicate to eliminate the check.
+ let GISelPredicateCode = [{return true;}];
+}
let isMoveImm = 1 in {
let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll
index ed1ee4527ed89..792f2aa711455 100644
--- a/llvm/test/CodeGen/AMDGPU/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshl.ll
@@ -49,12 +49,15 @@ define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_mov_b32 s4, s1
+; GFX9-NEXT: s_mov_b32 s5, s0
+; GFX9-NEXT: s_lshr_b32 s3, s0, 1
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
; GFX9-NEXT: s_not_b32 s2, s2
-; GFX9-NEXT: s_lshr_b32 s1, s0, 1
-; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_alignbit_b32 v1, s1, v1, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -77,13 +80,18 @@ define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX10-NEXT: s_lshr_b32 s0, s0, 1
-; GFX10-NEXT: s_not_b32 s1, s2
-; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s1
+; GFX10-NEXT: s_mov_b32 s5, s0
+; GFX10-NEXT: s_lshr_b32 s3, s0, 1
+; GFX10-NEXT: s_not_b32 s2, s2
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_and_b32 s2, s2, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshl_i32:
@@ -91,14 +99,18 @@ define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_not_b32 s1, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11-NEXT: s_mov_b32 s6, s1
+; GFX11-NEXT: s_mov_b32 s7, s0
+; GFX11-NEXT: s_lshr_b32 s3, s0, 1
+; GFX11-NEXT: s_not_b32 s2, s2
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_and_b32 s2, s2, 31
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
@@ -136,8 +148,10 @@ define amdgpu_kernel void @fshl_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v1, s2, v1, 25
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s2
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], 25
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -158,16 +172,22 @@ define amdgpu_kernel void @fshl_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s3, 25
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s2
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 25
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshl_i32_imm:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, s2, s3, 25
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], 25
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
entry:
@@ -230,18 +250,23 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x3c
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: s_lshr_b32 s3, s1, 1
-; GFX9-NEXT: v_alignbit_b32 v0, s1, v0, 1
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_lshr_b32 s10, s1, 1
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
; GFX9-NEXT: s_not_b32 s1, s9
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_alignbit_b32 v1, s3, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: s_not_b32 s1, s8
-; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX9-NEXT: s_lshr_b32 s0, s0, 1
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v3
+; GFX9-NEXT: s_mov_b32 s5, s10
+; GFX9-NEXT: s_and_b32 s1, s1, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_lshr_b32 s5, s0, 1
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
+; GFX9-NEXT: s_not_b32 s2, s8
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -271,14 +296,23 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX10-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v0, s1, s3, 1
-; GFX10-NEXT: v_alignbit_b32 v3, s0, s2, 1
-; GFX10-NEXT: s_lshr_b32 s1, s1, 1
-; GFX10-NEXT: s_not_b32 s2, s7
-; GFX10-NEXT: s_lshr_b32 s0, s0, 1
-; GFX10-NEXT: s_not_b32 s3, s6
-; GFX10-NEXT: v_alignbit_b32 v1, s1, v0, s2
-; GFX10-NEXT: v_alignbit_b32 v0, s0, v3, s3
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_lshr_b32 s10, s1, 1
+; GFX10-NEXT: s_not_b32 s7, s7
+; GFX10-NEXT: s_lshr_b32 s11, s0, 1
+; GFX10-NEXT: s_not_b32 s6, s6
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_and_b32 s4, s7, 31
+; GFX10-NEXT: s_and_b32 s5, s6, 31
+; GFX10-NEXT: s_mov_b32 s3, s11
+; GFX10-NEXT: s_mov_b32 s1, s10
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX10-NEXT: s_endpgm
;
@@ -288,16 +322,25 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, s1, s3, 1
-; GFX11-NEXT: v_alignbit_b32 v3, s0, s2, 1
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_not_b32 s2, s7
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_not_b32 s3, s6
-; GFX11-NEXT: v_alignbit_b32 v1, s1, v0, s2
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v3, s3
+; GFX11-NEXT: s_mov_b32 s8, s3
+; GFX11-NEXT: s_mov_b32 s9, s1
+; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_lshr_b32 s10, s1, 1
+; GFX11-NEXT: s_not_b32 s7, s7
+; GFX11-NEXT: s_lshr_b32 s11, s0, 1
+; GFX11-NEXT: s_not_b32 s6, s6
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_and_b32 s7, s7, 31
+; GFX11-NEXT: s_and_b32 s6, s6, 31
+; GFX11-NEXT: s_mov_b32 s3, s11
+; GFX11-NEXT: s_mov_b32 s1, s10
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX11-NEXT: s_endpgm
entry:
@@ -341,10 +384,13 @@ define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_alignbit_b32 v1, s1, v0, 23
-; GFX9-NEXT: v_alignbit_b32 v0, s0, v3, 25
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 23
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 25
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -369,8 +415,13 @@ define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s1, s3, 23
-; GFX10-NEXT: v_alignbit_b32 v0, s0, s2, 25
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], 25
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 23
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT: s_endpgm
;
@@ -379,10 +430,15 @@ define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, s1, s3, 23
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s2, 25
+; GFX11-NEXT: s_mov_b32 s6, s3
+; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b32 s7, s1
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 25
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 23
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
; GFX11-NEXT: s_endpgm
entry:
@@ -465,34 +521,44 @@ define amdgpu_kernel void @fshl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX9-LABEL: fshl_v4i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s4, s15
+; GFX9-NEXT: s_mov_b32 s5, s11
+; GFX9-NEXT: s_lshr_b32 s16, s11, 1
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
; GFX9-NEXT: s_not_b32 s3, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s15
-; GFX9-NEXT: s_lshr_b32 s4, s11, 1
-; GFX9-NEXT: v_alignbit_b32 v0, s11, v0, 1
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v3, s4, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s14
-; GFX9-NEXT: s_not_b32 s2, s2
-; GFX9-NEXT: v_alignbit_b32 v0, s10, v0, 1
+; GFX9-NEXT: s_mov_b32 s5, s16
+; GFX9-NEXT: s_and_b32 s3, s3, 31
+; GFX9-NEXT: s_mov_b32 s15, s10
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s3
; GFX9-NEXT: s_lshr_b32 s3, s10, 1
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_alignbit_b32 v2, s3, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[14:15], 1
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_mov_b32 s11, s3
+; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[10:11], s2
+; GFX9-NEXT: s_mov_b32 s10, s13
+; GFX9-NEXT: s_mov_b32 s11, s9
+; GFX9-NEXT: s_lshr_b32 s3, s9, 1
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[10:11], 1
; GFX9-NEXT: s_not_b32 s1, s1
-; GFX9-NEXT: v_alignbit_b32 v0, s9, v0, 1
-; GFX9-NEXT: s_lshr_b32 s2, s9, 1
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_alignbit_b32 v1, s2, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
-; GFX9-NEXT: s_not_b32 s0, s0
-; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 1
+; GFX9-NEXT: s_mov_b32 s11, s3
+; GFX9-NEXT: s_and_b32 s1, s1, 31
+; GFX9-NEXT: s_mov_b32 s13, s8
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[10:11], s1
; GFX9-NEXT: s_lshr_b32 s1, s8, 1
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_alignbit_b32 v0, s1, v0, v5
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX9-NEXT: s_not_b32 s0, s0
+; GFX9-NEXT: s_mov_b32 s9, s1
+; GFX9-NEXT: s_and_b32 s0, s0, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s10
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -530,22 +596,40 @@ define amdgpu_kernel void @fshl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v0, s11, s15, 1
-; GFX10-NEXT: v_alignbit_b32 v1, s10, s14, 1
-; GFX10-NEXT: v_alignbit_b32 v5, s9, s13, 1
-; GFX10-NEXT: v_alignbit_b32 v6, s8, s12, 1
-; GFX10-NEXT: s_lshr_b32 s4, s11, 1
-; GFX10-NEXT: s_not_b32 s3, s3
-; GFX10-NEXT: s_lshr_b32 s5, s10, 1
-; GFX10-NEXT: s_not_b32 s2, s2
-; GFX10-NEXT: s_lshr_b32 s9, s9, 1
+; GFX10-NEXT: s_mov_b32 s4, s15
+; GFX10-NEXT: s_mov_b32 s5, s11
+; GFX10-NEXT: s_mov_b32 s15, s10
+; GFX10-NEXT: s_lshr_b32 s16, s11, 1
+; GFX10-NEXT: s_not_b32 s11, s3
+; GFX10-NEXT: s_lshr_b32 s17, s10, 1
+; GFX10-NEXT: s_not_b32 s10, s2
+; GFX10-NEXT: s_lshr_b32 s18, s9, 1
+; GFX10-NEXT: s_mov_b32 s2, s13
+; GFX10-NEXT: s_mov_b32 s3, s9
+; GFX10-NEXT: s_lshr_b32 s19, s8, 1
+; GFX10-NEXT: s_mov_b32 s13, s8
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[14:15], 1
+; GFX10-NEXT: s_and_b32 s11, s11, 31
+; GFX10-NEXT: s_and_b32 s10, s10, 31
+; GFX10-NEXT: s_mov_b32 s5, s16
+; GFX10-NEXT: s_mov_b32 s9, s17
; GFX10-NEXT: s_not_b32 s1, s1
-; GFX10-NEXT: s_lshr_b32 s8, s8, 1
; GFX10-NEXT: s_not_b32 s0, s0
-; GFX10-NEXT: v_alignbit_b32 v3, s4, v0, s3
-; GFX10-NEXT: v_alignbit_b32 v2, s5, v1, s2
-; GFX10-NEXT: v_alignbit_b32 v1, s9, v5, s1
-; GFX10-NEXT: v_alignbit_b32 v0, s8, v6, s0
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s11
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s10
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[12:13], 1
+; GFX10-NEXT: s_mov_b32 s3, s18
+; GFX10-NEXT: s_mov_b32 s11, s19
+; GFX10-NEXT: s_and_b32 s0, s0, 31
+; GFX10-NEXT: s_and_b32 s5, s1, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[10:11], s0
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, s8
+; GFX10-NEXT: v_mov_b32_e32 v3, s4
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX10-NEXT: s_endpgm
;
@@ -555,24 +639,41 @@ define amdgpu_kernel void @fshl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, s11, s15, 1
-; GFX11-NEXT: v_alignbit_b32 v1, s10, s14, 1
-; GFX11-NEXT: v_alignbit_b32 v5, s9, s13, 1
-; GFX11-NEXT: v_alignbit_b32 v6, s8, s12, 1
-; GFX11-NEXT: s_lshr_b32 s6, s11, 1
-; GFX11-NEXT: s_not_b32 s3, s3
-; GFX11-NEXT: s_lshr_b32 s7, s10, 1
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: s_lshr_b32 s9, s9, 1
+; GFX11-NEXT: s_mov_b32 s6, s15
+; GFX11-NEXT: s_mov_b32 s7, s11
+; GFX11-NEXT: s_mov_b32 s15, s10
+; GFX11-NEXT: s_lshr_b32 s16, s11, 1
+; GFX11-NEXT: s_not_b32 s11, s3
+; GFX11-NEXT: s_lshr_b32 s17, s10, 1
+; GFX11-NEXT: s_not_b32 s10, s2
+; GFX11-NEXT: s_lshr_b32 s18, s9, 1
+; GFX11-NEXT: s_mov_b32 s2, s13
+; GFX11-NEXT: s_mov_b32 s3, s9
+; GFX11-NEXT: s_lshr_b32 s19, s8, 1
+; GFX11-NEXT: s_mov_b32 s13, s8
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[14:15], 1
+; GFX11-NEXT: s_and_b32 s11, s11, 31
+; GFX11-NEXT: s_and_b32 s10, s10, 31
+; GFX11-NEXT: s_mov_b32 s7, s16
+; GFX11-NEXT: s_mov_b32 s9, s17
; GFX11-NEXT: s_not_b32 s1, s1
-; GFX11-NEXT: s_lshr_b32 s8, s8, 1
; GFX11-NEXT: s_not_b32 s0, s0
-; GFX11-NEXT: v_alignbit_b32 v3, s6, v0, s3
-; GFX11-NEXT: v_alignbit_b32 v2, s7, v1, s2
-; GFX11-NEXT: v_alignbit_b32 v1, s9, v5, s1
-; GFX11-NEXT: v_alignbit_b32 v0, s8, v6, s0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s11
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s10
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[12:13], 1
+; GFX11-NEXT: s_mov_b32 s3, s18
+; GFX11-NEXT: s_mov_b32 s11, s19
+; GFX11-NEXT: s_and_b32 s0, s0, 31
+; GFX11-NEXT: s_and_b32 s7, s1, 31
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[10:11], s0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s6
+; GFX11-NEXT: v_mov_b32_e32 v2, s8
; GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX11-NEXT: s_endpgm
entry:
@@ -624,14 +725,20 @@ define amdgpu_kernel void @fshl_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s15
-; GFX9-NEXT: v_mov_b32_e32 v1, s14
-; GFX9-NEXT: v_alignbit_b32 v3, s11, v0, 31
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_alignbit_b32 v2, s10, v1, 23
-; GFX9-NEXT: v_alignbit_b32 v1, s9, v0, 25
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
-; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 31
+; GFX9-NEXT: s_mov_b32 s2, s15
+; GFX9-NEXT: s_mov_b32 s3, s11
+; GFX9-NEXT: s_mov_b32 s15, s10
+; GFX9-NEXT: s_mov_b32 s6, s13
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s13, s8
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[14:15], 23
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], 25
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[12:13], 31
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -660,10 +767,20 @@ define amdgpu_kernel void @fshl_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v3, s11, s15, 31
-; GFX10-NEXT: v_alignbit_b32 v2, s10, s14, 23
-; GFX10-NEXT: v_alignbit_b32 v1, s9, s13, 25
-; GFX10-NEXT: v_alignbit_b32 v0, s8, s12, 31
+; GFX10-NEXT: s_mov_b32 s2, s15
+; GFX10-NEXT: s_mov_b32 s3, s11
+; GFX10-NEXT: s_mov_b32 s15, s10
+; GFX10-NEXT: s_mov_b32 s4, s13
+; GFX10-NEXT: s_mov_b32 s5, s9
+; GFX10-NEXT: s_mov_b32 s13, s8
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 31
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[14:15], 23
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[12:13], 31
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 25
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -672,12 +789,21 @@ define amdgpu_kernel void @fshl_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v3, s11, s15, 31
-; GFX11-NEXT: v_alignbit_b32 v2, s10, s14, 23
-; GFX11-NEXT: v_alignbit_b32 v1, s9, s13, 25
-; GFX11-NEXT: v_alignbit_b32 v0, s8, s12, 31
+; GFX11-NEXT: s_mov_b32 s2, s15
+; GFX11-NEXT: s_mov_b32 s3, s11
+; GFX11-NEXT: s_mov_b32 s15, s10
+; GFX11-NEXT: s_mov_b32 s4, s13
+; GFX11-NEXT: s_mov_b32 s5, s9
+; GFX11-NEXT: s_mov_b32 s13, s8
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 31
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[14:15], 23
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[12:13], 31
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 25
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 4fccebabce04f..a32d4d8dcda88 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -55,9 +55,11 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, v2
+; GFX9-NEXT: s_mov_b32 s4, s1
+; GFX9-NEXT: s_mov_b32 s5, s0
+; GFX9-NEXT: s_and_b32 s0, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -77,68 +79,45 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s1
+; GFX10-NEXT: s_mov_b32 s5, s0
+; GFX10-NEXT: s_and_b32 s0, s2, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_i32:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_i32:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, s0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s2, 31
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_i32:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
-; GFX12-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX12-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_i32:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
-; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, s1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, s0
-; GFX12-FAKE16-NEXT: s_and_b32 s0, s2, 31
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[4:5]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_i32:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s6, s1
+; GFX11-NEXT: s_mov_b32 s7, s0
+; GFX11-NEXT: s_and_b32 s0, s2, 31
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_i32:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s6, s1
+; GFX12-NEXT: s_mov_b32 s7, s0
+; GFX12-NEXT: s_and_b32 s0, s2, 31
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: global_store_b32 v0, v1, s[4:5]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
store i32 %0, ptr addrspace(1) %in
@@ -175,8 +154,10 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v1, s2, v1, 7
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s2
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -197,57 +178,136 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s3, 7
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s2
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_i32_imm:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s2, s3, 7
-; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_i32_imm:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_i32_imm:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s2, s3, 7
-; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_i32_imm:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_i32_imm:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_i32_imm:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s4, s3
+; GFX12-NEXT: s_mov_b32 s5, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 7)
store i32 %0, ptr addrspace(1) %in
ret void
}
+define amdgpu_kernel void @fshr_i32_imm_src0(ptr addrspace(1) %in, i32 %x, i32 %y) {
+; SI-LABEL: fshr_i32_imm_src0:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_alignbit_b32 v0, 7, s3, v0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: fshr_i32_imm_src0:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_alignbit_b32 v2, 7, s3, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: fshr_i32_imm_src0:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s5, 7
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; R600-LABEL: fshr_i32_imm_src0:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T1.X, literal.x, KC0[2].W, KC0[2].Z,
+; R600-NEXT: 7(9.809089e-45), 0(0.000000e+00)
+;
+; GFX10-LABEL: fshr_i32_imm_src0:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s5, 7
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_and_b32 s2, s2, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: fshr_i32_imm_src0:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s5, 7
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_and_b32 s2, s2, 31
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_i32_imm_src0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s5, 7
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s4, s3
+; GFX12-NEXT: s_and_b32 s2, s2, 31
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
+entry:
+ %0 = call i32 @llvm.fshr.i32(i32 7, i32 %y, i32 %x)
+ store i32 %0, ptr addrspace(1) %in
+ ret void
+}
+
define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
; SI-LABEL: fshr_v2i32:
; SI: ; %bb.0: ; %entry
@@ -290,12 +350,15 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_alignbit_b32 v1, s1, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v3
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_and_b32 s1, s7, 31
+; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_and_b32 s0, s6, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX9-NEXT: s_endpgm
;
@@ -316,89 +379,62 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX10-LABEL: fshr_v2i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
; GFX10-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s6
-; GFX10-NEXT: v_alignbit_b32 v1, s1, s3, v0
-; GFX10-NEXT: v_alignbit_b32 v0, s0, s2, v2
-; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[8:9]
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_and_b32 s0, s6, 31
+; GFX10-NEXT: s_and_b32 s6, s7, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s6
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_v2i32:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x2
-; GFX11-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, s6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, v0.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, v0.h
-; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_v2i32:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x2
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s6, 31
-; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 31
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_v2i32:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_clause 0x2
-; GFX12-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, v0.l
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, v0.h
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_v2i32:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_clause 0x2
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s1
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, s0
-; GFX12-FAKE16-NEXT: s_and_b32 s0, s6, 31
-; GFX12-FAKE16-NEXT: s_and_b32 s6, s7, 31
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_v2i32:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s8, s3
+; GFX11-NEXT: s_mov_b32 s9, s1
+; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_and_b32 s0, s6, 31
+; GFX11-NEXT: s_and_b32 s6, s7, 31
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v2i32:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x2
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s8, s3
+; GFX12-NEXT: s_mov_b32 s9, s1
+; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_and_b32 s0, s6, 31
+; GFX12-NEXT: s_and_b32 s6, s7, 31
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
store <2 x i32> %0, ptr addrspace(1) %in
@@ -440,10 +476,13 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_alignbit_b32 v1, s1, v0, 9
-; GFX9-NEXT: v_alignbit_b32 v0, s0, v3, 7
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 9
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 7
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -468,74 +507,182 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s1, s3, 9
-; GFX10-NEXT: v_alignbit_b32 v0, s0, s2, 7
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 9
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_v2i32_imm:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, 9
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, 7
-; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_v2i32_imm:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s0
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, s1
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_v2i32_imm:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s1, s3, 9
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s0, s2, 7
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_v2i32_imm:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, s0
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, s1
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_v2i32_imm:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s6, s3
+; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b32 s7, s1
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v2i32_imm:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s6, s3
+; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b32 s7, s1
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
store <2 x i32> %0, ptr addrspace(1) %in
ret void
}
+define amdgpu_kernel void @fshr_v2i32_imm_src1(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
+; SI-LABEL: fshr_v2i32_imm_src1:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_alignbit_b32 v1, s1, 9, v0
+; SI-NEXT: v_alignbit_b32 v0, s0, 7, v2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: fshr_v2i32_imm_src1:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s3
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_alignbit_b32 v1, s1, 9, v0
+; VI-NEXT: v_alignbit_b32 v0, s0, 7, v2
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: fshr_v2i32_imm_src1:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s4, 9
+; GFX9-NEXT: s_mov_b32 s8, 7
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_and_b32 s1, s3, 31
+; GFX9-NEXT: s_mov_b32 s9, s0
+; GFX9-NEXT: s_and_b32 s0, s2, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_endpgm
+;
+; R600-LABEL: fshr_v2i32_imm_src1:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: BIT_ALIGN_INT * T0.Y, KC0[3].X, literal.x, KC0[3].Z,
+; R600-NEXT: 9(1.261169e-44), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T0.X, KC0[2].W, literal.x, KC0[3].Y,
+; R600-NEXT: 7(9.809089e-45), 0(0.000000e+00)
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX10-LABEL: fshr_v2i32_imm_src1:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s4, 9
+; GFX10-NEXT: s_mov_b32 s8, 7
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: s_mov_b32 s9, s0
+; GFX10-NEXT: s_and_b32 s0, s2, 31
+; GFX10-NEXT: s_and_b32 s2, s3, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: fshr_v2i32_imm_src1:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s6, 9
+; GFX11-NEXT: s_mov_b32 s8, 7
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s7, s1
+; GFX11-NEXT: s_mov_b32 s9, s0
+; GFX11-NEXT: s_and_b32 s0, s2, 31
+; GFX11-NEXT: s_and_b32 s2, s3, 31
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v2i32_imm_src1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s6, 9
+; GFX12-NEXT: s_mov_b32 s8, 7
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s7, s1
+; GFX12-NEXT: s_mov_b32 s9, s0
+; GFX12-NEXT: s_and_b32 s0, s2, 31
+; GFX12-NEXT: s_and_b32 s2, s3, 31
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: s_endpgm
+entry:
+ %0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> <i32 7, i32 9>, <2 x i32> %y)
+ store <2 x i32> %0, ptr addrspace(1) %in
+ ret void
+}
+
define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; SI-LABEL: fshr_v4i32:
; SI: ; %bb.0: ; %entry
@@ -590,18 +737,24 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s15
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v3, s11, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s14
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_alignbit_b32 v2, s10, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_alignbit_b32 v1, s9, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, v5
+; GFX9-NEXT: s_mov_b32 s4, s15
+; GFX9-NEXT: s_mov_b32 s5, s11
+; GFX9-NEXT: s_and_b32 s3, s3, 31
+; GFX9-NEXT: s_mov_b32 s15, s10
+; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_mov_b32 s10, s13
+; GFX9-NEXT: s_mov_b32 s11, s9
+; GFX9-NEXT: s_and_b32 s1, s1, 31
+; GFX9-NEXT: s_mov_b32 s13, s8
+; GFX9-NEXT: s_and_b32 s0, s0, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s3
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[14:15], s2
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[10:11], s1
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[12:13], s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s10
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX9-NEXT: s_endpgm
;
@@ -626,119 +779,87 @@ define amdgpu_kernel void @fshr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX10-LABEL: fshr_v4i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
+; GFX10-NEXT: s_mov_b32 s4, s15
+; GFX10-NEXT: s_mov_b32 s5, s11
+; GFX10-NEXT: s_and_b32 s11, s3, 31
+; GFX10-NEXT: s_mov_b32 s15, s10
+; GFX10-NEXT: s_and_b32 s10, s2, 31
+; GFX10-NEXT: s_mov_b32 s2, s13
+; GFX10-NEXT: s_mov_b32 s3, s9
+; GFX10-NEXT: s_and_b32 s16, s1, 31
+; GFX10-NEXT: s_mov_b32 s13, s8
+; GFX10-NEXT: s_and_b32 s8, s0, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], s11
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[14:15], s10
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s1
-; GFX10-NEXT: v_mov_b32_e32 v5, s0
-; GFX10-NEXT: v_alignbit_b32 v3, s11, s15, v0
-; GFX10-NEXT: v_alignbit_b32 v2, s10, s14, v1
-; GFX10-NEXT: v_alignbit_b32 v1, s9, s13, v4
-; GFX10-NEXT: v_alignbit_b32 v0, s8, s12, v5
-; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: v_mov_b32_e32 v3, s0
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_v4i32:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x2
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
-; GFX11-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, v0.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, v1.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, v4.l
-; GFX11-TRUE16-NEXT: global_store_b128 v5, v[0:3], s[4:5]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_v4i32:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x2
-; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s15
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, s11
-; GFX11-FAKE16-NEXT: s_and_b32 s11, s3, 31
-; GFX11-FAKE16-NEXT: s_mov_b32 s15, s10
-; GFX11-FAKE16-NEXT: s_and_b32 s10, s2, 31
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, s13
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s9
-; GFX11-FAKE16-NEXT: s_and_b32 s16, s1, 31
-; GFX11-FAKE16-NEXT: s_mov_b32 s13, s8
-; GFX11-FAKE16-NEXT: s_and_b32 s8, s0, 31
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_v4i32:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_clause 0x2
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
-; GFX12-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX12-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, v0.l
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, v1.l
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, v4.l
-; GFX12-TRUE16-NEXT: global_store_b128 v5, v[0:3], s[4:5]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_v4i32:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_clause 0x2
-; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
-; GFX12-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, s15
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, s11
-; GFX12-FAKE16-NEXT: s_and_b32 s11, s3, 31
-; GFX12-FAKE16-NEXT: s_mov_b32 s15, s10
-; GFX12-FAKE16-NEXT: s_and_b32 s10, s2, 31
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, s13
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, s9
-; GFX12-FAKE16-NEXT: s_and_b32 s16, s1, 31
-; GFX12-FAKE16-NEXT: s_mov_b32 s13, s8
-; GFX12-FAKE16-NEXT: s_and_b32 s8, s0, 31
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[4:5]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_v4i32:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s6, s15
+; GFX11-NEXT: s_mov_b32 s7, s11
+; GFX11-NEXT: s_and_b32 s11, s3, 31
+; GFX11-NEXT: s_mov_b32 s15, s10
+; GFX11-NEXT: s_and_b32 s10, s2, 31
+; GFX11-NEXT: s_mov_b32 s2, s13
+; GFX11-NEXT: s_mov_b32 s3, s9
+; GFX11-NEXT: s_and_b32 s16, s1, 31
+; GFX11-NEXT: s_mov_b32 s13, s8
+; GFX11-NEXT: s_and_b32 s8, s0, 31
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v4i32:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x2
+; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x54
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s6, s15
+; GFX12-NEXT: s_mov_b32 s7, s11
+; GFX12-NEXT: s_and_b32 s11, s3, 31
+; GFX12-NEXT: s_mov_b32 s15, s10
+; GFX12-NEXT: s_and_b32 s10, s2, 31
+; GFX12-NEXT: s_mov_b32 s2, s13
+; GFX12-NEXT: s_mov_b32 s3, s9
+; GFX12-NEXT: s_and_b32 s16, s1, 31
+; GFX12-NEXT: s_mov_b32 s13, s8
+; GFX12-NEXT: s_and_b32 s8, s0, 31
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[6:7], s11
+; GFX12-NEXT: s_lshr_b64 s[6:7], s[14:15], s10
+; GFX12-NEXT: s_lshr_b64 s[8:9], s[12:13], s8
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], s16
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX12-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z)
store <4 x i32> %0, ptr addrspace(1) %in
@@ -788,14 +909,20 @@ define amdgpu_kernel void @fshr_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s15
-; GFX9-NEXT: v_mov_b32_e32 v1, s14
-; GFX9-NEXT: v_alignbit_b32 v3, s11, v0, 1
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_alignbit_b32 v2, s10, v1, 9
-; GFX9-NEXT: v_alignbit_b32 v1, s9, v0, 7
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
-; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 1
+; GFX9-NEXT: s_mov_b32 s2, s15
+; GFX9-NEXT: s_mov_b32 s3, s11
+; GFX9-NEXT: s_mov_b32 s15, s10
+; GFX9-NEXT: s_mov_b32 s6, s13
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s13, s8
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[14:15], 9
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], 7
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -822,92 +949,242 @@ define amdgpu_kernel void @fshr_v4i32_imm(ptr addrspace(1) %in, <4 x i32> %x, <4
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v3, s11, s15, 1
-; GFX10-NEXT: v_alignbit_b32 v2, s10, s14, 9
-; GFX10-NEXT: v_alignbit_b32 v1, s9, s13, 7
-; GFX10-NEXT: v_alignbit_b32 v0, s8, s12, 1
+; GFX10-NEXT: s_mov_b32 s2, s15
+; GFX10-NEXT: s_mov_b32 s3, s11
+; GFX10-NEXT: s_mov_b32 s15, s10
+; GFX10-NEXT: s_mov_b32 s4, s13
+; GFX10-NEXT: s_mov_b32 s5, s9
+; GFX10-NEXT: s_mov_b32 s13, s8
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: fshr_v4i32_imm:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, 1
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, 9
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, 7
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, 1
-; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: fshr_v4i32_imm:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, s15
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s11
-; GFX11-FAKE16-NEXT: s_mov_b32 s15, s10
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s13
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s9
-; GFX11-FAKE16-NEXT: s_mov_b32 s13, s8
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-FAKE16-NEXT: s_endpgm
-;
-; GFX12-TRUE16-LABEL: fshr_v4i32_imm:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX12-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v3, s11, s15, 1
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v2, s10, s14, 9
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v1, s9, s13, 7
-; GFX12-TRUE16-NEXT: v_alignbit_b32 v0, s8, s12, 1
-; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: fshr_v4i32_imm:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, s15
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, s11
-; GFX12-FAKE16-NEXT: s_mov_b32 s15, s10
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s13
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s9
-; GFX12-FAKE16-NEXT: s_mov_b32 s13, s8
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
-; GFX12-FAKE16-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: fshr_v4i32_imm:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s2, s15
+; GFX11-NEXT: s_mov_b32 s3, s11
+; GFX11-NEXT: s_mov_b32 s15, s10
+; GFX11-NEXT: s_mov_b32 s4, s13
+; GFX11-NEXT: s_mov_b32 s5, s9
+; GFX11-NEXT: s_mov_b32 s13, s8
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v4i32_imm:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s2, s15
+; GFX12-NEXT: s_mov_b32 s3, s11
+; GFX12-NEXT: s_mov_b32 s15, s10
+; GFX12-NEXT: s_mov_b32 s4, s13
+; GFX12-NEXT: s_mov_b32 s5, s9
+; GFX12-NEXT: s_mov_b32 s13, s8
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX12-NEXT: s_lshr_b64 s[6:7], s[14:15], 9
+; GFX12-NEXT: s_lshr_b64 s[8:9], s[12:13], 1
+; GFX12-NEXT: s_lshr_b64 s[4:5], s[4:5], 7
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%0 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 1, i32 7, i32 9, i32 33>)
store <4 x i32> %0, ptr addrspace(1) %in
ret void
}
+define amdgpu_kernel void @fshr_v4i32_imm_src0(ptr addrspace(1) %in, <4 x i32> %x, <4 x i32> %y) {
+; SI-LABEL: fshr_v4i32_imm_src0:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s15
+; SI-NEXT: v_mov_b32_e32 v1, s14
+; SI-NEXT: v_alignbit_b32 v3, 33, s11, v0
+; SI-NEXT: v_mov_b32_e32 v0, s13
+; SI-NEXT: v_alignbit_b32 v2, 9, s10, v1
+; SI-NEXT: v_alignbit_b32 v1, 7, s9, v0
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_alignbit_b32 v0, 1, s8, v0
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: fshr_v4i32_imm_src0:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s15
+; VI-NEXT: v_mov_b32_e32 v1, s14
+; VI-NEXT: v_mov_b32_e32 v4, s13
+; VI-NEXT: v_alignbit_b32 v3, 33, s11, v0
+; VI-NEXT: v_alignbit_b32 v2, 9, s10, v1
+; VI-NEXT: v_alignbit_b32 v1, 7, s9, v4
+; VI-NEXT: v_mov_b32_e32 v0, s12
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_alignbit_b32 v0, 1, s8, v0
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: fshr_v4i32_imm_src0:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GFX9-NEXT: s_mov_b32 s1, 33
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 7
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s0, s11
+; GFX9-NEXT: s_and_b32 s4, s15, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; GFX9-NEXT: s_mov_b32 s11, 9
+; GFX9-NEXT: s_and_b32 s1, s14, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[10:11], s1
+; GFX9-NEXT: s_mov_b32 s6, s9
+; GFX9-NEXT: s_and_b32 s1, s13, 31
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], s1
+; GFX9-NEXT: s_mov_b32 s9, 1
+; GFX9-NEXT: s_and_b32 s1, s12, 31
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[8:9], s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; R600-LABEL: fshr_v4i32_imm_src0:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 8, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: BIT_ALIGN_INT * T0.W, literal.x, KC0[4].X, KC0[5].X,
+; R600-NEXT: 33(4.624285e-44), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T0.Z, literal.x, KC0[3].W, KC0[4].W,
+; R600-NEXT: 9(1.261169e-44), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T0.Y, literal.x, KC0[3].Z, KC0[4].Z,
+; R600-NEXT: 7(9.809089e-45), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T0.X, 1, KC0[3].Y, KC0[4].Y,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX10-LABEL: fshr_v4i32_imm_src0:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s1, 33
+; GFX10-NEXT: s_mov_b32 s3, 7
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s0, s11
+; GFX10-NEXT: s_and_b32 s4, s15, 31
+; GFX10-NEXT: s_mov_b32 s11, 9
+; GFX10-NEXT: s_and_b32 s5, s14, 31
+; GFX10-NEXT: s_mov_b32 s2, s9
+; GFX10-NEXT: s_and_b32 s13, s13, 31
+; GFX10-NEXT: s_mov_b32 s9, 1
+; GFX10-NEXT: s_and_b32 s12, s12, 31
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[10:11], s5
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s12
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s13
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: v_mov_b32_e32 v3, s0
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: fshr_v4i32_imm_src0:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s1, 33
+; GFX11-NEXT: s_mov_b32 s3, 7
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, s11
+; GFX11-NEXT: s_and_b32 s6, s15, 31
+; GFX11-NEXT: s_mov_b32 s11, 9
+; GFX11-NEXT: s_and_b32 s7, s14, 31
+; GFX11-NEXT: s_mov_b32 s2, s9
+; GFX11-NEXT: s_and_b32 s13, s13, 31
+; GFX11-NEXT: s_mov_b32 s9, 1
+; GFX11-NEXT: s_and_b32 s12, s12, 31
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[10:11], s7
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s12
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s13
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fshr_v4i32_imm_src0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s1, 33
+; GFX12-NEXT: s_mov_b32 s3, 7
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s0, s11
+; GFX12-NEXT: s_and_b32 s6, s15, 31
+; GFX12-NEXT: s_mov_b32 s11, 9
+; GFX12-NEXT: s_and_b32 s7, s14, 31
+; GFX12-NEXT: s_mov_b32 s2, s9
+; GFX12-NEXT: s_and_b32 s13, s13, 31
+; GFX12-NEXT: s_mov_b32 s9, 1
+; GFX12-NEXT: s_and_b32 s12, s12, 31
+; GFX12-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
+; GFX12-NEXT: s_lshr_b64 s[6:7], s[10:11], s7
+; GFX12-NEXT: s_lshr_b64 s[8:9], s[8:9], s12
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], s13
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s0
+; GFX12-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX12-NEXT: s_endpgm
+entry:
+ %0 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> <i32 1, i32 7, i32 9, i32 33>, <4 x i32> %x, <4 x i32> %y)
+ store <4 x i32> %0, ptr addrspace(1) %in
+ ret void
+}
+
define i32 @v_fshr_i32(i32 %src0, i32 %src1, i32 %src2) {
; GFX89-LABEL: v_fshr_i32:
; GFX89: ; %bb.0:
>From 418c8fbcb62c8dd9b4fa629f1c7fb78fbc0bde95 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 4 Nov 2025 13:03:05 -0600
Subject: [PATCH 5/5] limit pattern to gfx9+; reduce multiple occurences of
same pattern
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 32 ++++++------------------
1 file changed, 8 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f71f7ea42d48e..ad3f9d7c90b70 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2701,14 +2701,6 @@ def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
->;
-
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
->;
-
} // isGFX9GFX10
} // end True16Predicate = NotHasTrue16BitInsts
@@ -2738,14 +2730,6 @@ def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
(EXTRACT_SUBREG VGPR_32:$src2, lo16),
/* clamp */ 0, /* op_sel */ 0)>;
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
->;
-
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
->;
-
} // end True16Predicate = UseRealTrue16Insts
let True16Predicate = UseFakeTrue16Insts in {
@@ -2782,14 +2766,6 @@ def : GCNPat<(DivergentTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
$src2, /* clamp */ 0, /* op_sel */ 0)
>;
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
->;
-
-def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
- (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
->;
-
} // end True16Predicate = UseFakeTrue16Insts
/********** ====================== **********/
@@ -3879,6 +3855,14 @@ class PackB32Pat<Instruction inst> : GCNPat <
>;
}
let SubtargetPredicate = isGFX9Plus in {
+ def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))
+ >;
+
+ def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),
+ (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))
+ >;
+
let True16Predicate = NotHasTrue16BitInsts in
def : PackB32Pat<V_PACK_B32_F16_e64>;
More information about the llvm-commits
mailing list