[llvm] d2b78c6 - AMDGPU: Custom lower bf16 shuffles (#122252)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 9 06:37:31 PST 2025
Author: Matt Arsenault
Date: 2025-01-09T21:37:27+07:00
New Revision: d2b78c646b9b7b63f1029ad1de954044ee6ebc32
URL: https://github.com/llvm/llvm-project/commit/d2b78c646b9b7b63f1029ad1de954044ee6ebc32
DIFF: https://github.com/llvm/llvm-project/commit/d2b78c646b9b7b63f1029ad1de954044ee6ebc32.diff
LOG: AMDGPU: Custom lower bf16 shuffles (#122252)
We already custom lower the other 16-bit element type shuffles.
Added:
Modified:
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0ac84f4e1f02af..992f7ed99d3bb7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -784,8 +784,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
{MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE,
- {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16,
- MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16},
+ {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
+ MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
+ MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
Custom);
for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
@@ -7545,9 +7546,8 @@ SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
SDLoc SL(Op);
EVT ResultVT = Op.getValueType();
ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
-
- EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16;
- EVT EltVT = PackVT.getVectorElementType();
+ MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
+ MVT PackVT = MVT::getVectorVT(EltVT, 2);
int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements();
// vector_shuffle <0,1,6,7> lhs, rhs
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index e408e83da1c298..e7ae9d831424cc 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GFX940 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11 %s
@@ -31,16 +32,27 @@ define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_234u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_234u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_mov_b32_e32 v0, v6
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_234u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_mov_b32_e32 v0, v4
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v6
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_234u:
; GFX10: ; %bb.0:
@@ -94,13 +106,22 @@ define <4 x half> @shuffle_v4f16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_u3u1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_u3u1:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_u3u1:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_u3u1:
; GFX10: ; %bb.0:
@@ -151,16 +172,27 @@ define <4 x half> @shuffle_v4f16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_3u6u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_3u6u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_3u6u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_3u6u:
; GFX10: ; %bb.0:
@@ -189,16 +221,27 @@ define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_3uu7:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_3uu7:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_3uu7:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_3uu7:
; GFX10: ; %bb.0:
@@ -227,16 +270,27 @@ define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_35u5:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v4, v5, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_35u5:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v4, v5, s4
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_35u5:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v4, v5, s0
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_35u5:
; GFX10: ; %bb.0:
@@ -263,17 +317,29 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_357u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_357u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v4, v6, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_357u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v1, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v4, v6, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_357u:
; GFX10: ; %bb.0:
@@ -432,13 +498,22 @@ define <4 x half> @shuffle_v4f16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_2301:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_2301:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_2301:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_2301:
; GFX10: ; %bb.0:
@@ -773,13 +848,22 @@ define <4 x half> @shuffle_v4f16_6723(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_6745:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_6745:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_6745:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_6745:
; GFX10: ; %bb.0:
@@ -833,16 +917,27 @@ define <4 x half> @shuffle_v4f16_6767(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_2356:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_2356:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_2356:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_2356:
; GFX10: ; %bb.0:
@@ -871,16 +966,27 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_5623:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_5623:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_5623:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_5623:
; GFX10: ; %bb.0:
@@ -987,17 +1093,29 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_5734:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_5734:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_5734:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_alignbit_b32 v1, v4, v6, 16
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_5734:
; GFX10: ; %bb.0:
@@ -1027,16 +1145,27 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4i16_2356:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4i16_2356:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4i16_2356:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4i16_2356:
; GFX10: ; %bb.0:
@@ -1101,15 +1230,25 @@ define <4 x i16> @shuffle_v4i16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %a
}
define <4 x half> @shuffle_v4f16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_0000:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_0000:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v0, v0, s4
+; GX900-NEXT: v_mov_b32_e32 v1, v0
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_0000:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v0, v0, s0
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_0000:
; GFX10: ; %bb.0:
@@ -1170,16 +1309,27 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_1100:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_mov_b32 s5, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v1, v1, s4
-; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_1100:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_mov_b32 s5, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v1, v1, s4
+; GX900-NEXT: v_perm_b32 v1, v1, v1, s5
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_1100:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_mov_b32 s1, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v2, v2, s0
+; GFX940-NEXT: v_perm_b32 v1, v2, v2, s1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_1100:
; GFX10: ; %bb.0:
@@ -1205,16 +1355,27 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_6161:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_6161:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0xffff
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4
+; GX900-NEXT: v_mov_b32_e32 v1, v0
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_6161:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0xffff
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_6161:
; GFX10: ; %bb.0:
@@ -1243,14 +1404,23 @@ define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_2333:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_2333:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v0, v0, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_2333:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_2333:
; GFX10: ; %bb.0:
@@ -1274,14 +1444,23 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_6667:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_6667:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v0, v0, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_6667:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_6667:
; GFX10: ; %bb.0:
@@ -1436,16 +1615,27 @@ define <4 x half> @shuffle_v8f16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrspace
}
define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v8f16_13_14_2_3:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8f16_13_14_2_3:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8f16_13_14_2_3:
; GFX10: ; %bb.0:
@@ -1474,14 +1664,23 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace
}
define <4 x half> @shuffle_v3f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v3f16_0122:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v1, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v3f16_0122:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v1, v1, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v3f16_0122:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v1, v1, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v3f16_0122:
; GFX10: ; %bb.0:
@@ -1535,20 +1734,35 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v6f16_452367:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GFX9-NEXT: global_load_dword v7, v[3:4], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v6f16_452367:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v6, v1
+; GX900-NEXT: v_mov_b32_e32 v5, v0
+; GX900-NEXT: v_mov_b32_e32 v4, v3
+; GX900-NEXT: v_mov_b32_e32 v3, v2
+; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
+; GX900-NEXT: global_load_dword v7, v[3:4], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v2, v7
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v6f16_452367:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v7, v1
+; GFX940-NEXT: v_mov_b32_e32 v6, v0
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_mov_b32_e32 v4, v2
+; GFX940-NEXT: global_load_dwordx3 v[0:2], v[6:7], off
+; GFX940-NEXT: global_load_dword v3, v[4:5], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_mov_b32_e32 v0, v2
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, v3
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v6f16_452367:
; GFX10: ; %bb.0:
@@ -1583,22 +1797,40 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1)
}
define amdgpu_kernel void @fma_shuffle_v2f16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) {
-; GFX9-LABEL: fma_shuffle_v2f16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX9-NEXT: v_lshlrev_b32_e32 v6, 3, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1]
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1]
-; GFX9-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1]
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0]
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5]
-; GFX9-NEXT: s_endpgm
+; GX900-LABEL: fma_shuffle_v2f16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GX900-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GX900-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GX900-NEXT: s_waitcnt lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1]
+; GX900-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
+; GX900-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5]
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1]
+; GX900-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1]
+; GX900-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0]
+; GX900-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
+; GX900-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5]
+; GX900-NEXT: s_endpgm
+;
+; GFX940-LABEL: fma_shuffle_v2f16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX940-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX940-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX940-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1]
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v6, s[6:7]
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1]
+; GFX940-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1]
+; GFX940-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0]
+; GFX940-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
+; GFX940-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7] sc0 sc1
+; GFX940-NEXT: s_endpgm
;
; GFX10-LABEL: fma_shuffle_v2f16:
; GFX10: ; %bb.0: ; %entry
@@ -1670,18 +1902,29 @@ entry:
}
define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4f16_0456:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4f16_0456:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1
+; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4f16_0456:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v6, v4, s0
+; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4f16_0456:
; GFX10: ; %bb.0:
@@ -1711,19 +1954,31 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %
}
define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_scalar_load_v8i32_0123:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX9-NEXT: s_endpgm
+; GX900-LABEL: shuffle_scalar_load_v8i32_0123:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GX900-NEXT: v_mov_b32_e32 v4, 0
+; GX900-NEXT: s_waitcnt lgkmcnt(0)
+; GX900-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GX900-NEXT: s_waitcnt lgkmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, s4
+; GX900-NEXT: v_mov_b32_e32 v1, s5
+; GX900-NEXT: v_mov_b32_e32 v2, s6
+; GX900-NEXT: v_mov_b32_e32 v3, s7
+; GX900-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
+; GX900-NEXT: s_endpgm
+;
+; GFX940-LABEL: shuffle_scalar_load_v8i32_0123:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX940-NEXT: v_mov_b32_e32 v4, 0
+; GFX940-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX940-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX940-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT: s_endpgm
;
; GFX10-LABEL: shuffle_scalar_load_v8i32_0123:
; GFX10: ; %bb.0:
@@ -1757,15 +2012,25 @@ define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in,
}
define <2 x half> @low16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: low16bits_v2f16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: low16bits_v2f16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: low16bits_v2f16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: low16bits_v2f16:
; GFX10: ; %bb.0: ; %entry
@@ -1793,15 +2058,25 @@ entry:
}
define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: hi16bits_v2f16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: hi16bits_v2f16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: hi16bits_v2f16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: hi16bits_v2f16:
; GFX10: ; %bb.0: ; %entry
@@ -1829,15 +2104,25 @@ entry:
}
define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: low16hi16bits_v2f16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: low16hi16bits_v2f16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0xffff
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: low16hi16bits_v2f16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0xffff
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_bfi_b32 v0, s0, v4, v5
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: low16hi16bits_v2f16:
; GFX10: ; %bb.0: ; %entry
@@ -1900,15 +2185,25 @@ entry:
}
define <2 x i16> @i16_low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: i16_low16bits:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: i16_low16bits:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: i16_low16bits:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: i16_low16bits:
; GFX10: ; %bb.0: ; %entry
@@ -1936,15 +2231,25 @@ entry:
}
define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: i16_low16hi16bits:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: i16_low16hi16bits:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0xffff
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: i16_low16hi16bits:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0xffff
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_bfi_b32 v0, s0, v4, v5
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: i16_low16hi16bits:
; GFX10: ; %bb.0: ; %entry
@@ -2007,15 +2312,25 @@ entry:
}
define <2 x i16> @i16_hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: i16_hi16bits:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: i16_hi16bits:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: i16_hi16bits:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: i16_hi16bits:
; GFX10: ; %bb.0: ; %entry
@@ -2107,15 +2422,25 @@ entry:
}
define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v8f16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8f16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8f16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8f16_concat:
; GFX10: ; %bb.0:
@@ -2142,17 +2467,29 @@ define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v16f16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v16f16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v16f16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v16f16_concat:
; GFX10: ; %bb.0:
@@ -2183,23 +2520,41 @@ define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
}
define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v32f16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
-; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
-; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v32f16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v32f16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v32f16_concat:
; GFX10: ; %bb.0:
@@ -2246,15 +2601,25 @@ define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
}
define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v8i16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8i16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8i16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8i16_concat:
; GFX10: ; %bb.0:
@@ -2281,17 +2646,29 @@ define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v16i16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v16i16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v16i16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v16i16_concat:
; GFX10: ; %bb.0:
@@ -2322,23 +2699,41 @@ define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
}
define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v32i16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
-; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
-; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v32i16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v32i16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v32i16_concat:
; GFX10: ; %bb.0:
@@ -2385,16 +2780,28 @@ define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
}
define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v4i8_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v0, v[0:1], off
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: global_load_short_d16_hi v0, v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dword v[4:5], v0, off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4i8_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_ushort v0, v[0:1], off
+; GX900-NEXT: s_nop 0
+; GX900-NEXT: global_load_short_d16_hi v0, v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dword v[4:5], v0, off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4i8_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_ushort v6, v[0:1], off
+; GFX940-NEXT: global_load_ushort v7, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v7, v6, s0
+; GFX940-NEXT: global_store_dword v[4:5], v0, off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4i8_concat:
; GFX10: ; %bb.0:
@@ -2421,15 +2828,25 @@ define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1,
}
define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v8i8_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v6, v[0:1], off
-; GFX9-NEXT: global_load_dword v7, v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v[4:5], v[6:7], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8i8_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v6, v[0:1], off
+; GX900-NEXT: global_load_dword v7, v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx2 v[4:5], v[6:7], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8i8_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v6, v[0:1], off
+; GFX940-NEXT: global_load_dword v7, v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx2 v[4:5], v[6:7], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8i8_concat:
; GFX10: ; %bb.0:
@@ -2456,15 +2873,25 @@ define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1,
}
define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v16i8_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v16i8_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v16i8_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v16i8_concat:
; GFX10: ; %bb.0:
@@ -2491,17 +2918,29 @@ define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v32i8_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v32i8_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v32i8_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v32i8_concat:
; GFX10: ; %bb.0:
@@ -2532,15 +2971,25 @@ define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v4i32_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4i32_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4i32_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4i32_concat:
; GFX10: ; %bb.0:
@@ -2567,17 +3016,29 @@ define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v8i32_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8i32_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8i32_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8i32_concat:
; GFX10: ; %bb.0:
@@ -2608,23 +3069,41 @@ define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
}
define void @shuffle_v16i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v16i32_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
-; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
-; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v16i32_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v16i32_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v16i32_concat:
; GFX10: ; %bb.0:
@@ -2698,16 +3177,27 @@ define <4 x bfloat> @shuffle_v4bf16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_234u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_234u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_mov_b32_e32 v0, v6
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_234u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_mov_b32_e32 v0, v4
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v6
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_234u:
; GFX10: ; %bb.0:
@@ -2761,13 +3251,22 @@ define <4 x bfloat> @shuffle_v4bf16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_u3u1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_u3u1:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_u3u1:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_u3u1:
; GFX10: ; %bb.0:
@@ -2818,16 +3317,27 @@ define <4 x bfloat> @shuffle_v4bf16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_3u6u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_3u6u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_3u6u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_3u6u:
; GFX10: ; %bb.0:
@@ -2856,16 +3366,27 @@ define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_3uu7:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_3uu7:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_3uu7:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_3uu7:
; GFX10: ; %bb.0:
@@ -2894,16 +3415,27 @@ define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_35u5:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v4, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v4, v5, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_35u5:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v4, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v4, v5, s4
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_35u5:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4
+; GFX940-NEXT: global_load_dword v4, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v4, v5, s0
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_35u5:
; GFX10: ; %bb.0:
@@ -2930,17 +3462,29 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_357u:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_357u:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v4, v6, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_357u:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v1, s0, v5, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v4, v6, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_357u:
; GFX10: ; %bb.0:
@@ -2973,29 +3517,24 @@ define <4 x bfloat> @shuffle_v4bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_0101:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_0101:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_0101:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3035,39 +3574,31 @@ define <4 x bfloat> @shuffle_v4bf16_0145(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_0145:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX9-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-NEXT: global_load_dword v5, v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_0145:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v5, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_0145:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
@@ -3079,33 +3610,30 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_0167:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
-; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_0167:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
-; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_0167:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3115,22 +3643,28 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_2301:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_2301:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_2301:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_2301:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -3139,7 +3673,6 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3182,33 +3715,30 @@ define <4 x bfloat> @shuffle_v4bf16_2345(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_2345:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v5, v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_2345:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX10-NEXT: global_load_dword v5, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_2345:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v2, v2
+; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3257,39 +3787,33 @@ define <4 x bfloat> @shuffle_v4bf16_4501(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_4501:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX9-NEXT: global_load_dword v4, v[2:3], off
+; GFX9-NEXT: global_load_dword v5, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_4501:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
-; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_4501:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b64 v[3:4], v[0:1], off
+; GFX11-NEXT: global_load_b32 v2, v[2:3], off
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v3, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
@@ -3301,33 +3825,32 @@ define <4 x bfloat> @shuffle_v4bf16_4523(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_4523:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v4, v[2:3], off
+; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_4523:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_4523:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v2, v[2:3], off
; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3340,29 +3863,24 @@ define <4 x bfloat> @shuffle_v4bf16_4545(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_4545:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v0, v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_4545:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: global_load_dword v0, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_4545:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: global_load_b32 v0, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3402,33 +3920,32 @@ define <4 x bfloat> @shuffle_v4bf16_6701(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v4bf16_6701:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v5, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_6701:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4
+; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v4bf16_6701:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
-; GFX11-NEXT: global_load_b32 v0, v[2:3], off offset:4
+; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v4, v4
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3476,22 +3993,28 @@ define <4 x bfloat> @shuffle_v4bf16_6723(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_6745:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_6745:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_6745:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v3
+; GFX940-NEXT: v_mov_b32_e32 v1, v2
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_6745:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -3500,7 +4023,6 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
@@ -3540,16 +4062,27 @@ define <4 x bfloat> @shuffle_v4bf16_6767(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_2356:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_2356:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v0, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_2356:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v0, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_2356:
; GFX10: ; %bb.0:
@@ -3578,16 +4111,27 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_5623:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_5623:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_5623:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_5623:
; GFX10: ; %bb.0:
@@ -3694,17 +4238,29 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_5734:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_5734:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_5734:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_alignbit_b32 v1, v4, v6, 16
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_5734:
; GFX10: ; %bb.0:
@@ -3734,15 +4290,25 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_0000:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_0000:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v0, v0, s4
+; GX900-NEXT: v_mov_b32_e32 v1, v0
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_0000:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v0, v0, s0
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_0000:
; GFX10: ; %bb.0:
@@ -3803,16 +4369,27 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_1100:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_mov_b32 s5, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v1, v1, s4
-; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_1100:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_mov_b32 s5, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v1, v1, s4
+; GX900-NEXT: v_perm_b32 v1, v1, v1, s5
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_1100:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_mov_b32 s1, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v2, v2, s0
+; GFX940-NEXT: v_perm_b32 v1, v2, v2, s1
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_1100:
; GFX10: ; %bb.0:
@@ -3838,16 +4415,27 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_6161:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_6161:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0xffff
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4
+; GX900-NEXT: v_mov_b32_e32 v1, v0
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_6161:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0xffff
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4
+; GFX940-NEXT: v_mov_b32_e32 v1, v0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_6161:
; GFX10: ; %bb.0:
@@ -3876,14 +4464,23 @@ define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_2333:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_2333:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v0, v0, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_2333:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_2333:
; GFX10: ; %bb.0:
@@ -3907,14 +4504,23 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_6667:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_6667:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v0, v0, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_6667:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_6667:
; GFX10: ; %bb.0:
@@ -3941,29 +4547,24 @@ define <4 x bfloat> @shuffle_v8bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v8bf16_0101:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8bf16_0101:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v8bf16_0101:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
@@ -4003,39 +4604,31 @@ define <4 x bfloat> @shuffle_v8bf16_4589(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX9-LABEL: shuffle_v8bf16_4589:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:8
+; GFX9-NEXT: global_load_dword v5, v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8bf16_4589:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8
-; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:8
+; GFX10-NEXT: global_load_dword v5, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v8bf16_4589:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:8
-; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
@@ -4082,16 +4675,27 @@ define <4 x bfloat> @shuffle_v8bf16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrsp
}
define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v8bf16_13_14_2_3:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8
-; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8bf16_13_14_2_3:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8
+; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v1, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8
+; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v1, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8bf16_13_14_2_3:
; GFX10: ; %bb.0:
@@ -4120,23 +4724,29 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp
}
define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v3bf16_0122:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_mov_b32 s5, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0
-; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v3bf16_0122:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v1, v1, v1, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v3bf16_0122:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v1, v1, v1, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v3bf16_0122:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -4145,7 +4755,6 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0
; GFX11-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <3 x bfloat>, ptr addrspace(1) %arg0
@@ -4185,20 +4794,35 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v6bf16_452367:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GFX9-NEXT: global_load_dword v7, v[3:4], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v6bf16_452367:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v6, v1
+; GX900-NEXT: v_mov_b32_e32 v5, v0
+; GX900-NEXT: v_mov_b32_e32 v4, v3
+; GX900-NEXT: v_mov_b32_e32 v3, v2
+; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
+; GX900-NEXT: global_load_dword v7, v[3:4], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_mov_b32_e32 v2, v7
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v6bf16_452367:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v7, v1
+; GFX940-NEXT: v_mov_b32_e32 v6, v0
+; GFX940-NEXT: v_mov_b32_e32 v5, v3
+; GFX940-NEXT: v_mov_b32_e32 v4, v2
+; GFX940-NEXT: global_load_dwordx3 v[0:2], v[6:7], off
+; GFX940-NEXT: global_load_dword v3, v[4:5], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_mov_b32_e32 v0, v2
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_mov_b32_e32 v2, v3
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v6bf16_452367:
; GFX10: ; %bb.0:
@@ -4233,88 +4857,173 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace
}
define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) {
-; GFX9-LABEL: fma_shuffle_v2bf16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT: s_movk_i32 s2, 0x7fff
-; GFX9-NEXT: s_mov_b32 s3, 0x7060302
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1]
-; GFX9-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5]
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7]
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_fma_f32 v7, v8, v9, v7
-; GFX9-NEXT: v_fma_f32 v1, v8, v5, v1
-; GFX9-NEXT: v_fma_f32 v2, v12, v5, v2
-; GFX9-NEXT: v_bfe_u32 v5, v7, 16, 1
-; GFX9-NEXT: v_fma_f32 v8, v12, v9, v11
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GFX9-NEXT: v_add3_u32 v5, v5, v7, s2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX9-NEXT: v_bfe_u32 v13, v8, 16, 1
-; GFX9-NEXT: v_add3_u32 v11, v11, v1, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v8
-; GFX9-NEXT: v_bfe_u32 v15, v2, 16, 1
-; GFX9-NEXT: v_add3_u32 v13, v13, v8, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
-; GFX9-NEXT: v_add3_u32 v15, v15, v2, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_fma_f32 v1, v3, v10, v1
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_fma_f32 v3, v3, v6, v5
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_fma_f32 v2, v4, v10, v2
-; GFX9-NEXT: v_fma_f32 v4, v4, v6, v7
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX9-NEXT: v_add3_u32 v5, v5, v1, s2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add3_u32 v7, v7, v3, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_bfe_u32 v11, v4, 16, 1
-; GFX9-NEXT: v_add3_u32 v9, v9, v2, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX9-NEXT: v_add3_u32 v11, v11, v4, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
-; GFX9-NEXT: v_perm_b32 v2, v4, v2, s3
-; GFX9-NEXT: v_perm_b32 v1, v3, v1, s3
-; GFX9-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GX900-LABEL: fma_shuffle_v2bf16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
+; GX900-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GX900-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GX900-NEXT: s_movk_i32 s2, 0x7fff
+; GX900-NEXT: s_mov_b32 s3, 0x7060302
+; GX900-NEXT: s_waitcnt lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1]
+; GX900-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5]
+; GX900-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7]
+; GX900-NEXT: s_waitcnt vmcnt(2)
+; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GX900-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GX900-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GX900-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GX900-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GX900-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GX900-NEXT: v_fma_f32 v7, v8, v9, v7
+; GX900-NEXT: v_fma_f32 v1, v8, v5, v1
+; GX900-NEXT: v_fma_f32 v2, v12, v5, v2
+; GX900-NEXT: v_bfe_u32 v5, v7, 16, 1
+; GX900-NEXT: v_fma_f32 v8, v12, v9, v11
+; GX900-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GX900-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GX900-NEXT: v_add3_u32 v5, v5, v7, s2
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GX900-NEXT: v_bfe_u32 v13, v8, 16, 1
+; GX900-NEXT: v_add3_u32 v11, v11, v1, s2
+; GX900-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GX900-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GX900-NEXT: v_bfe_u32 v15, v2, 16, 1
+; GX900-NEXT: v_add3_u32 v13, v13, v8, s2
+; GX900-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GX900-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GX900-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GX900-NEXT: v_or_b32_e32 v16, 0x400000, v2
+; GX900-NEXT: v_add3_u32 v15, v15, v2, s2
+; GX900-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GX900-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GX900-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GX900-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc
+; GX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GX900-NEXT: v_fma_f32 v1, v3, v10, v1
+; GX900-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GX900-NEXT: v_fma_f32 v3, v3, v6, v5
+; GX900-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GX900-NEXT: v_fma_f32 v2, v4, v10, v2
+; GX900-NEXT: v_fma_f32 v4, v4, v6, v7
+; GX900-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GX900-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GX900-NEXT: v_add3_u32 v5, v5, v1, s2
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GX900-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GX900-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GX900-NEXT: v_add3_u32 v7, v7, v3, s2
+; GX900-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GX900-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GX900-NEXT: v_bfe_u32 v11, v4, 16, 1
+; GX900-NEXT: v_add3_u32 v9, v9, v2, s2
+; GX900-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GX900-NEXT: v_add3_u32 v11, v11, v4, s2
+; GX900-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GX900-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
+; GX900-NEXT: v_perm_b32 v2, v4, v2, s3
+; GX900-NEXT: v_perm_b32 v1, v3, v1, s3
+; GX900-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
+; GX900-NEXT: s_endpgm
+;
+; GFX940-LABEL: fma_shuffle_v2bf16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX940-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX940-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX940-NEXT: s_movk_i32 s2, 0x7fff
+; GFX940-NEXT: s_mov_b32 s3, 0x7060302
+; GFX940-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1]
+; GFX940-NEXT: global_load_dwordx2 v[2:3], v6, s[8:9]
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11]
+; GFX940-NEXT: s_waitcnt vmcnt(2)
+; GFX940-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX940-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX940-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX940-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX940-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX940-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX940-NEXT: v_fmac_f32_e32 v1, v12, v4
+; GFX940-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX940-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX940-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX940-NEXT: v_add3_u32 v4, v4, v7, s2
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX940-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX940-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX940-NEXT: v_add3_u32 v9, v9, v0, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX940-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX940-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX940-NEXT: v_add3_u32 v13, v13, v11, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX940-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX940-NEXT: v_or_b32_e32 v16, 0x400000, v1
+; GFX940-NEXT: v_add3_u32 v15, v15, v1, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX940-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX940-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc
+; GFX940-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX940-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX940-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX940-NEXT: v_fmac_f32_e32 v4, v2, v5
+; GFX940-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX940-NEXT: v_fmac_f32_e32 v1, v3, v10
+; GFX940-NEXT: v_fmac_f32_e32 v7, v3, v5
+; GFX940-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX940-NEXT: v_add3_u32 v2, v2, v0, s2
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX940-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX940-NEXT: v_add3_u32 v5, v5, v4, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX940-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX940-NEXT: v_add3_u32 v9, v9, v1, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX940-NEXT: v_or_b32_e32 v12, 0x400000, v7
+; GFX940-NEXT: v_add3_u32 v11, v11, v7, s2
+; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
+; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX940-NEXT: v_perm_b32 v0, v2, v0, s3
+; GFX940-NEXT: s_nop 0
+; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v12, vcc
+; GFX940-NEXT: v_perm_b32 v1, v3, v1, s3
+; GFX940-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1] sc0 sc1
+; GFX940-NEXT: s_endpgm
;
; GFX10-LABEL: fma_shuffle_v2bf16:
; GFX10: ; %bb.0: ; %entry
@@ -4523,18 +5232,29 @@ entry:
}
define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
-; GFX9-LABEL: shuffle_v4bf16_0456:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v4bf16_0456:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1
+; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v4bf16_0456:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v6, v4, s0
+; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v4bf16_0456:
; GFX10: ; %bb.0:
@@ -4564,15 +5284,25 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1
}
define <2 x bfloat> @low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: low16bits:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: low16bits:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x5040100
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: low16bits:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x5040100
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: low16bits:
; GFX10: ; %bb.0: ; %entry
@@ -4600,15 +5330,25 @@ entry:
}
define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: hi16bits_v2bf16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v5, v[2:3], off
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: hi16bits_v2bf16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[0:1], off
+; GX900-NEXT: global_load_dword v5, v[2:3], off
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_perm_b32 v0, v5, v4, s4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: hi16bits_v2bf16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[0:1], off
+; GFX940-NEXT: global_load_dword v5, v[2:3], off
+; GFX940-NEXT: s_mov_b32 s0, 0x7060302
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: hi16bits_v2bf16:
; GFX10: ; %bb.0: ; %entry
@@ -4636,15 +5376,25 @@ entry:
}
define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
-; GFX9-LABEL: low16hi16bits_v2bf16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v4, v[2:3], off
-; GFX9-NEXT: global_load_dword v5, v[0:1], off
-; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: low16hi16bits_v2bf16:
+; GX900: ; %bb.0: ; %entry
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dword v4, v[2:3], off
+; GX900-NEXT: global_load_dword v5, v[0:1], off
+; GX900-NEXT: s_mov_b32 s4, 0xffff
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: low16hi16bits_v2bf16:
+; GFX940: ; %bb.0: ; %entry
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dword v4, v[2:3], off
+; GFX940-NEXT: global_load_dword v5, v[0:1], off
+; GFX940-NEXT: s_mov_b32 s0, 0xffff
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: low16hi16bits_v2bf16:
; GFX10: ; %bb.0: ; %entry
@@ -4739,15 +5489,25 @@ entry:
}
define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v8bf16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v8bf16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v8bf16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v8bf16_concat:
; GFX10: ; %bb.0:
@@ -4774,17 +5534,29 @@ define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
}
define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v16bf16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v16bf16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(1)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v16bf16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(1)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v16bf16_concat:
; GFX10: ; %bb.0:
@@ -4815,23 +5587,41 @@ define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %ar
}
define void @shuffle_v32bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) {
-; GFX9-LABEL: shuffle_v32bf16_concat:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
-; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
-; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GX900-LABEL: shuffle_v32bf16_concat:
+; GX900: ; %bb.0:
+; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
+; GX900-NEXT: s_waitcnt vmcnt(3)
+; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16
+; GX900-NEXT: s_waitcnt vmcnt(0)
+; GX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shuffle_v32bf16_concat:
+; GFX940: ; %bb.0:
+; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
+; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off
+; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(3)
+; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1
+; GFX940-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: shuffle_v32bf16_concat:
; GFX10: ; %bb.0:
More information about the llvm-commits
mailing list