[llvm] fdaa2d0 - [AMDGPU] Use divergent addresses for vector loads
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 23 05:33:42 PST 2021
Author: Jay Foad
Date: 2021-02-23T13:33:15Z
New Revision: fdaa2d02591b10c96ca8705041dfc75fcbf97095
URL: https://github.com/llvm/llvm-project/commit/fdaa2d02591b10c96ca8705041dfc75fcbf97095
DIFF: https://github.com/llvm/llvm-project/commit/fdaa2d02591b10c96ca8705041dfc75fcbf97095.diff
LOG: [AMDGPU] Use divergent addresses for vector loads
Change some test cases to use divergent addresses for vector loads,
which should be the common case in real world code. Using uniform
addresses causes poor instruction selection for the surrounding
code which has to be fixed up post-register-allocation, and this causes
a lot of testsuite churn for a forthcoming patch to stop selecting
24-bit vector multiply instructions for uniform multiplies.
This shows up some problems in the idot tests where we fail to select
v_dot instructions because the patterns only match MUL_[UI]24 ISD nodes,
but the DAG contains i16 mul nodes instead.
Differential Revision: https://reviews.llvm.org/D97062
Added:
Modified:
llvm/test/CodeGen/AMDGPU/idot2.ll
llvm/test/CodeGen/AMDGPU/idot4s.ll
llvm/test/CodeGen/AMDGPU/idot4u.ll
llvm/test/CodeGen/AMDGPU/idot8s.ll
llvm/test/CodeGen/AMDGPU/idot8u.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/idot2.ll b/llvm/test/CodeGen/AMDGPU/idot2.ll
index dd813b0e0e44..dc114d468378 100644
--- a/llvm/test/CodeGen/AMDGPU/idot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot2.ll
@@ -14,24 +14,28 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -39,21 +43,27 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -63,22 +73,17 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -86,38 +91,40 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -146,24 +153,29 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: v_mul_u32_u24_e32 v0, s5, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: v_mul_u32_u24_e32 v0, v0, v2
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, s5, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -171,21 +183,25 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s2, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, s2, v0
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s5, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -195,22 +211,17 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v1, s5, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, s9, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, s0
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -218,51 +229,46 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mul_u32_u24_e32 v1, s5, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, s9, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, s0
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_MulMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s5, s0, s4
-; GFX10-DL-NEXT: s_and_b32 s4, s1, s4
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-DL-NEXT: v_mul_u32_u24_e64 v0, s4, s5
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, s8, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, s2
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -288,21 +294,26 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i16 s7, s4
-; GFX7-NEXT: s_ashr_i32 s4, s4, 16
-; GFX7-NEXT: s_sext_i32_i16 s8, s5
-; GFX7-NEXT: s_ashr_i32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -310,20 +321,26 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_ashr_i32 s2, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_ashr_i32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -333,21 +350,17 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -355,38 +368,40 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot2_i32_i16 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot2_i32_i16 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot2_i32_i16 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot2_i32_i16 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -413,21 +428,26 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s7, s4, 16
-; GFX7-NEXT: s_lshr_b32 s8, s5, 16
-; GFX7-NEXT: s_sext_i32_i16 s4, s4
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1
-; GFX7-NEXT: s_sext_i32_i16 s5, s5
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_i32_i24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -435,20 +455,26 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mad_u32_u24 v0, s3, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -458,21 +484,17 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -480,49 +502,46 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2_MixedTypedMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -548,24 +567,28 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -573,21 +596,27 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -597,22 +626,22 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
+; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v4, v3, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -620,38 +649,40 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_alt_AddOperands:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -678,21 +709,26 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i16 s7, s4
-; GFX7-NEXT: s_ashr_i32 s4, s4, 16
-; GFX7-NEXT: s_and_b32 s8, s5, 0xffff
-; GFX7-NEXT: s_ashr_i32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -700,20 +736,26 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_ashr_i32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX8-NEXT: s_ashr_i32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -723,21 +765,17 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, 0xffff
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -745,49 +783,46 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-DL-NEXT: s_and_b32 s5, s1, 0xffff
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2_MixedExt:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16
-; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16
-; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-DL-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -814,17 +849,24 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s7, s[0:1], 0x0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s5, s6, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, s5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, s4, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v0, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -832,16 +874,24 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v2, v[2:3]
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, s2, v0
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, s4, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s2
+; GFX8-NEXT: v_mad_u32_u24 v2, v0, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -851,17 +901,18 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, s1, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, s0, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, s0, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -869,42 +920,48 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, s1, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, s0, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, s0, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notudot2_SameVec:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s1, s8
-; GFX10-DL-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -930,24 +987,29 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 16
-; GFX7-NEXT: s_lshr_b32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -955,21 +1017,27 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[2:3]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v3, s2, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v1, v0, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v3, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -979,22 +1047,17 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1002,38 +1065,40 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_v4i16:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
- %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -1059,24 +1124,28 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x1
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 16
-; GFX7-NEXT: s_lshr_b32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:4
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:4
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1084,21 +1153,31 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x4
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x4
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s6, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v4
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v3, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v3, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1108,22 +1187,17 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x4
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x4
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5] offset:4
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7] offset:4
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1131,38 +1205,40 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x4
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x4
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5] offset:4
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7] offset:4
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_v4i16_Hi:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x4
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x4
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5] offset:4
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7] offset:4
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
- %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 2
%conv = zext i16 %s1.elt1 to i32
@@ -1188,24 +1264,29 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s6, s6, s8
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
+; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1213,21 +1294,27 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s8, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s3, s8
-; GFX8-NEXT: s_and_b32 s2, s2, s8
-; GFX8-NEXT: s_and_b32 s5, s5, s8
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0
-; GFX8-NEXT: s_and_b32 s4, s4, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1237,74 +1324,64 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s10, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s11, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s1, s1, s10
-; GFX9-NODL-NEXT: s_and_b32 s0, s0, s10
-; GFX9-NODL-NEXT: s_and_b32 s5, s9, s10
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
-; GFX9-NODL-NEXT: s_and_b32 s4, s8, s10
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v0
+; GFX9-NODL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: notudot2_v4i16_Even:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s10, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s11, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s1, s1, s10
-; GFX9-DL-NEXT: s_and_b32 s0, s0, s10
-; GFX9-DL-NEXT: s_and_b32 s5, s9, s10
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
-; GFX9-DL-NEXT: s_and_b32 s4, s8, s10
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v0
+; GFX9-DL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notudot2_v4i16_Even:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s10, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s10
-; GFX10-DL-NEXT: s_and_b32 s1, s1, s4
-; GFX10-DL-NEXT: s_and_b32 s5, s9, s4
-; GFX10-DL-NEXT: s_and_b32 s0, s0, s4
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s1, v0
-; GFX10-DL-NEXT: s_and_b32 s1, s8, s4
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
- %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -1330,24 +1407,29 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: s_lshr_b32 s6, s6, 16
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
+; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1355,21 +1437,27 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s8, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s3, s8
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s5, s5, s8
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1379,74 +1467,64 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s10, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s11, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s1, s1, s10
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s9, s10
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
-; GFX9-NODL-NEXT: s_lshr_b32 s4, s8, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v0
+; GFX9-NODL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: notudot2_v4i16_Middle:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s10, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s11, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s1, s1, s10
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_and_b32 s5, s9, s10
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
-; GFX9-DL-NEXT: s_lshr_b32 s4, s8, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v0
+; GFX9-DL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notudot2_v4i16_Middle:
-; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s10, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s10
-; GFX10-DL-NEXT: s_and_b32 s1, s1, s4
-; GFX10-DL-NEXT: s_and_b32 s4, s9, s4
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s1, v0
-; GFX10-DL-NEXT: s_lshr_b32 s1, s8, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
- %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 1
%conv = zext i16 %s1.elt1 to i32
@@ -1472,24 +1550,28 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v2, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1497,21 +1579,27 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, v1, v0
-; GFX8-NEXT: s_lshr_b32 s7, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1521,22 +1609,17 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-NODL-NEXT: s_and_b32 s1, s1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1544,51 +1627,46 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-DL-NEXT: s_and_b32 s1, s1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notudot2_DiffIndex:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_lshr_b32 s5, s0, 16
-; GFX10-DL-NEXT: s_and_b32 s6, s1, s4
-; GFX10-DL-NEXT: s_and_b32 s0, s0, s4
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -1614,24 +1692,28 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v1, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s5
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
@@ -1640,22 +1722,28 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v1, s2, v1, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
+; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1665,23 +1753,19 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v2, v1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1689,53 +1773,51 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v2, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_MultipleUses_add1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s6, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX10-DL-NEXT: s_and_b32 s0, s0, s6
-; GFX10-DL-NEXT: s_and_b32 s1, s1, s6
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
-; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -1764,22 +1846,27 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i16 s7, s4
-; GFX7-NEXT: s_ashr_i32 s4, s4, 16
-; GFX7-NEXT: s_sext_i32_i16 s8, s5
-; GFX7-NEXT: s_ashr_i32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mad_i32_i24 v1, s8, v1, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_i32_i24 v1, v3, v1, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1787,21 +1874,27 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_ashr_i32 s2, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_ashr_i32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v1, s6, v2, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v1, v2, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1811,22 +1904,19 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s5, v2, v1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1834,51 +1924,51 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_i32_i24 v2, s5, v2, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2_MultipleUses_add1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16
-; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16
-; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v1, s1, s0, v0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
-; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v0
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -1906,25 +1996,29 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v2, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v1, s5, v0, v1
-; GFX7-NEXT: v_mad_u32_u24 v1, s7, v2, v1
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
+; GFX7-NEXT: v_mad_u32_u24 v4, v0, v2, s4
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1932,22 +2026,28 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v2, v0
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v4, v2, v1, s3
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v4
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1957,23 +2057,22 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v4, v3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
+; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v3, s1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1981,53 +2080,56 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v2
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v4, v3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
+; GFX9-DL-NEXT: v_mad_u32_u24 v3, v4, v3, s1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_MultipleUses_mul1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_mov_b32 s3, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_and_b32_e32 v3, s3, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_and_b32 s5, s0, s4
-; GFX10-DL-NEXT: s_and_b32 s4, s1, s4
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v2
+; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -2057,22 +2159,27 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i16 s7, s4
-; GFX7-NEXT: s_sext_i32_i16 s8, s5
-; GFX7-NEXT: s_ashr_i32 s4, s4, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_ashr_i32 s5, s5, 16
-; GFX7-NEXT: v_mad_i32_i24 v1, s8, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s4
-; GFX7-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v0, v1
+; GFX7-NEXT: v_mad_i32_i24 v4, v3, v1, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v4
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2080,21 +2187,27 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_ashr_i32 s2, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_ashr_i32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v2, v0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mad_i32_i24 v4, v2, v1, s2
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v4
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -2104,22 +2217,21 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 16
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v4, v3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v3, v1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v4, v3, s0
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -2127,51 +2239,54 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 16
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, v4, v3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v3, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_mad_i32_i24 v3, v4, v3, s0
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2_MultipleUses_mul1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 16
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX10-DL-NEXT: s_sext_i32_i16 s5, s1
-; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v2
+; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -2200,25 +2315,29 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v1, s7, v0, v1
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v4, v3, v1, s4
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2226,22 +2345,28 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s3, s2
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v4, v0, v3, s3
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v4
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -2251,23 +2376,20 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v4, v2, v1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s1, v1, v2
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
+; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -2275,53 +2397,53 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v2, v1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v1, v2
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
+; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_MultipleUses_mul2:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_mov_b32 s6, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX10-DL-NEXT: s_and_b32 s0, s0, s6
-; GFX10-DL-NEXT: s_and_b32 s1, s1, s6
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v1
+; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32
@@ -2351,22 +2473,27 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i16 s7, s4
-; GFX7-NEXT: s_ashr_i32 s4, s4, 16
-; GFX7-NEXT: s_sext_i32_i16 s8, s5
-; GFX7-NEXT: s_ashr_i32 s5, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: v_mad_i32_i24 v1, s5, v0, v1
-; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v4, v0, v2, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v4
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2374,21 +2501,27 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i16 s5, s2
-; GFX8-NEXT: s_ashr_i32 s2, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_ashr_i32 s3, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT: v_mad_i32_i24 v4, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v4
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -2398,22 +2531,20 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v4, v2, v1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
+; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -2421,51 +2552,53 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v4, v2, v1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
+; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
+; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot2_MultipleUses_mul2:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16
-; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16
-; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
+; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v1
+; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32
@@ -2495,22 +2628,27 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 16
-; GFX7-NEXT: s_lshr_b32 s7, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_and_b32 s4, s4, s8
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v4, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2518,83 +2656,98 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_mov_b32 s0, 0xffff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s2, s0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ushort v6, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v1, v1, v5, v6
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot2_acc16:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s8, 16
-; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0
-; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ushort v5, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot2_acc16:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ushort v5, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_acc16:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot2_u32_u16 v1, s6, s7, v1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v0, v7, v5, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %v1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
- %v2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
+ %v1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
+ %v2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%v1e1 = extractelement <2 x i16> %v1, i64 0
%v2e1 = extractelement <2 x i16> %v2, i64 0
@@ -2615,56 +2768,57 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
; GFX7-LABEL: notsdot2_sext8:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_mov_b32 s10, s2
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s0, s4
-; GFX7-NEXT: s_mov_b32 s1, s5
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s7
-; GFX7-NEXT: s_mov_b32 s6, s2
-; GFX7-NEXT: s_mov_b32 s7, s3
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: buffer_load_ushort v1, off, s[4:7], 0
-; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_ushort v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_bfe_i32 v2, v0, 0, 8
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v3, v1, 0, 8
+; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 8
-; GFX7-NEXT: v_bfe_i32 v1, v1, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_i32_i24 v0, v1, v0, s0
-; GFX7-NEXT: v_mad_i32_i24 v0, v3, v2, v0
-; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: notsdot2_sext8:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_ushort v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
-; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
+; GFX8-NEXT: v_lshrrev_b16_e32 v3, 8, v3
+; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 8, v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfe_i32 v3, v1, 0, 8
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v0, v1, v0, s2
-; GFX8-NEXT: v_mad_i32_i24 v2, v3, v2, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -2674,22 +2828,19 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 8
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 8
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v2, 8, v2
-; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v4, v3, v1
+; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -2697,29 +2848,26 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 8
-; GFX9-DL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 8
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v2, 8, v2
-; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, v4, v3, v1
+; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notsdot2_sext8:
; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
@@ -2727,23 +2875,24 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
; GFX10-DL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v1
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, v2
-; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mad_i32_i24 v3, v4, v3, s2
-; GFX10-DL-NEXT: v_mad_i32_i24 v1, v2, v1, v3
-; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, s2, v1
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<2 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %src1
- %vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i8> %vec1, i64 0
%conv = sext i8 %s1.elt1 to i32
@@ -2763,3 +2912,5 @@ entry:
store i32 %add6, i32 addrspace(1)* %dst, align 4
ret void
}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index e8bdaff7f37b..53e261dc2e47 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -12,29 +12,32 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
+; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i8 s6, s4
-; GFX7-NEXT: s_sext_i32_i8 s7, s5
-; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_ashr_i32 s5, s5, 24
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT: s_ashr_i32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, s4
+; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_i32_i24 v1, v3, v6, v1
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -42,28 +45,32 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i8 s4, s2
-; GFX8-NEXT: s_sext_i32_i8 s5, s3
-; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
-; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010
-; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_ashr_i32 s3, s3, 24
-; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 24
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
+; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 8
+; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 8
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
+; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 8
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s2
+; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
+; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -73,29 +80,20 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s6, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -103,38 +101,40 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot4_i32_i8 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc32:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot4_i32_i8 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i32
@@ -177,38 +177,41 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i8 s6, s4
-; GFX7-NEXT: s_sext_i32_i8 s7, s5
-; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT: s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT: s_and_b32 s10, s10, s8
-; GFX7-NEXT: s_and_b32 s6, s6, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT: s_ashr_i32 s5, s5, 24
-; GFX7-NEXT: s_and_b32 s12, s12, s8
-; GFX7-NEXT: s_and_b32 s9, s9, s8
-; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: s_ashr_i32 s4, s4, 24
-; GFX7-NEXT: s_and_b32 s11, s11, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: v_mov_b32_e32 v3, s12
-; GFX7-NEXT: s_and_b32 s4, s4, s8
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
+; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -216,97 +219,154 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i8 s2, s0
-; GFX8-NEXT: s_sext_i32_i8 s3, s1
-; GFX8-NEXT: s_bfe_i32 s5, s1, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_bfe_i32 s7, s1, 0x80010
-; GFX8-NEXT: s_bfe_i32 s4, s0, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
-; GFX8-NEXT: s_bfe_i32 s6, s0, 0x80010
-; GFX8-NEXT: s_ashr_i32 s1, s1, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: s_ashr_i32 s0, s0, 24
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s4, v4, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_ushort v1, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v4
+; GFX8-NEXT: v_bfe_i32 v7, v4, 0, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX8-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v0
+; GFX8-NEXT: v_bfe_i32 v8, v0, 0, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mad_u16 v1, v7, v8, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX8-NEXT: v_mad_u16 v1, v9, v10, v1
+; GFX8-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX8-NEXT: v_mad_u16 v1, v5, v6, v1
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: idot4_acc16:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 8, v3
+; GFX9-NODL-NEXT: v_bfe_i32 v6, v2, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v7, v3, 0, 8
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v2, v1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s6, v3, v1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v6, v7, v4
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-NODL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v8, v9, v4
+; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot4_acc16:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 8, v3
+; GFX9-DL-NEXT: v_bfe_i32 v6, v2, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v7, v3, 0, 8
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-DL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot4_i32_i8 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v6, v7, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v8, v9, v4
+; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc16:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot4_i32_i8 v1, s6, s7, v1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX10-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v10, v2, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX10-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_bfe_i32 v4, v8, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v10, v9, 0, 8
+; GFX10-DL-NEXT: v_mad_u16 v3, v5, v6, v3
+; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
+; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i16
@@ -347,30 +407,33 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s7, s4, s8
-; GFX7-NEXT: s_and_b32 s6, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -378,99 +441,121 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX8-NEXT: s_and_b32 s3, s2, s0
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x80008
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_bfe_u32 s6, s2, 0x80010
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX8-NEXT: s_lshr_b32 s2, s2, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
+; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: idot4_acc8:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s8, 0x80010
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot4_acc8:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc8:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v4, v2, v3, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0
@@ -503,30 +588,33 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
+; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
+; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i8 s6, s4
-; GFX7-NEXT: s_sext_i32_i8 s7, s5
-; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
-; GFX7-NEXT: v_mad_i32_i24 v1, s6, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s9
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010
-; GFX7-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_ashr_i32 s5, s5, 24
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT: s_ashr_i32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v8, v1, v5, s4
+; GFX7-NEXT: v_mad_i32_i24 v3, v3, v6, v8
+; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, v3
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -534,29 +622,33 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
+; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 8
+; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 8
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
+; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sext_i32_i8 s4, s2
-; GFX8-NEXT: s_sext_i32_i8 s5, s3
-; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008
-; GFX8-NEXT: v_mad_i32_i24 v1, s4, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
-; GFX8-NEXT: v_mad_i32_i24 v1, s6, v2, v1
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010
-; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_ashr_i32 s3, s3, 24
-; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 24
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT: v_mad_i32_i24 v8, v1, v2, s2
+; GFX8-NEXT: v_mad_i32_i24 v4, v4, v5, v8
+; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, v4
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
+; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -566,30 +658,24 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 8
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 8
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v3, v4
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s6, v3, v2
-; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v3, v4, s0
+; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -597,65 +683,60 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 8
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v4
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-DL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x80008
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x80008
-; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x80010
-; GFX9-DL-NEXT: v_mad_i32_i24 v2, s6, v3, v2
-; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x80010
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-DL-NEXT: v_mad_i32_i24 v3, v3, v4, s0
+; GFX9-DL-NEXT: v_add3_u32 v2, v5, v3, v2
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_multiuse_mul1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 8
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 8
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v7, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_i32_i24_e32 v5, v0, v3
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_sext_i32_i8 s4, s0
-; GFX10-DL-NEXT: s_sext_i32_i8 s5, s1
-; GFX10-DL-NEXT: s_bfe_i32 s6, s0, 0x80008
-; GFX10-DL-NEXT: s_bfe_i32 s7, s1, 0x80008
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s6, s7, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
-; GFX10-DL-NEXT: s_bfe_i32 s4, s0, 0x80010
-; GFX10-DL-NEXT: s_bfe_i32 s5, s1, 0x80010
-; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 24
-; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v3, s2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v7, v0, v5
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i32
@@ -699,29 +780,32 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 24, v2
+; GFX7-NEXT: v_bfe_i32 v3, v2, 16, 8
+; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 8
+; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_ashrrev_i32_e32 v5, 24, v0
+; GFX7-NEXT: v_bfe_i32 v6, v0, 16, 8
+; GFX7-NEXT: v_bfe_i32 v7, v0, 8, 8
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_ashr_i32 s6, s4, 24
-; GFX7-NEXT: s_ashr_i32 s9, s5, 24
-; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80010
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80008
-; GFX7-NEXT: s_sext_i32_i8 s5, s5
-; GFX7-NEXT: s_bfe_i32 s7, s4, 0x80010
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
-; GFX7-NEXT: s_sext_i32_i8 s4, s4
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: v_mad_i32_i24 v0, s7, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v4, v7, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v6, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v1, v5, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -729,29 +813,34 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s8, s[0:1], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s2
-; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s3
-; GFX8-NEXT: s_ashr_i32 s6, s3, 24
-; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80010
-; GFX8-NEXT: s_sext_i32_i8 s3, s3
-; GFX8-NEXT: s_ashr_i32 s4, s2, 24
-; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80010
-; GFX8-NEXT: s_sext_i32_i8 s2, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_mov_b32_e32 v3, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v4, 24, v3
+; GFX8-NEXT: v_bfe_i32 v5, v3, 16, 8
+; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v6, 24, v0
+; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v2, v3
-; GFX8-NEXT: v_mad_i32_i24 v0, v0, v1, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_mad_i32_i24 v0, s5, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_i32_i24 v2, s4, v1, v0
+; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mad_i32_i24 v0, v3, v0, s2
+; GFX8-NEXT: v_mad_i32_i24 v0, v1, v2, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v5, v7, v0
+; GFX8-NEXT: v_mad_i32_i24 v2, v4, v6, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -761,30 +850,23 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s6, s1, 24
-; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80010
-; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s1
-; GFX9-NODL-NEXT: s_ashr_i32 s4, s0, 24
-; GFX9-NODL-NEXT: s_bfe_i32 s5, s0, 0x80010
-; GFX9-NODL-NEXT: s_sext_i32_i8 s0, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-NODL-NEXT: v_mad_i32_i24 v3, s0, v3, v4
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v1, v2, v3
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v5, s0, v2
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -792,66 +874,58 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-DL-NEXT: s_ashr_i32 s6, s1, 24
-; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x80010
-; GFX9-DL-NEXT: s_sext_i32_i8 s1, s1
-; GFX9-DL-NEXT: s_ashr_i32 s4, s0, 24
-; GFX9-DL-NEXT: s_bfe_i32 s5, s0, 0x80010
-; GFX9-DL-NEXT: s_sext_i32_i8 s0, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-DL-NEXT: v_mad_i32_i24 v3, s0, v3, v4
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, v1, v2, v3
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v2, v1
+; GFX9-DL-NEXT: v_add3_u32 v2, v5, s0, v2
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v7, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, s0
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s1
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-DL-NEXT: s_sext_i32_i8 s4, s0
-; GFX10-DL-NEXT: s_sext_i32_i8 s5, s1
-; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2
-; GFX10-DL-NEXT: s_bfe_i32 s4, s0, 0x80010
-; GFX10-DL-NEXT: s_bfe_i32 s5, s1, 0x80010
-; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 24
-; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 24
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v1, v2
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_i32_i24 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v7, s2, v0
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = sext <4 x i8> %vec1 to <4 x i32>
%cvec2 = sext <4 x i8> %vec2 to <4 x i32>
@@ -878,29 +952,47 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_ashr_i32 s6, s4, 24
-; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80010
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80008
-; GFX7-NEXT: s_ashr_i32 s9, s5, 24
-; GFX7-NEXT: s_sext_i32_i8 s5, s5
-; GFX7-NEXT: s_bfe_i32 s7, s4, 0x80010
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
-; GFX7-NEXT: s_sext_i32_i8 s4, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s11
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 8, 8
+; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 8
+; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8
+; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v6
+; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
+; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v7
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v2, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s7, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v8
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v7, v1
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v6, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -908,70 +1000,73 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_e64 v3, 8, s0
-; GFX8-NEXT: v_lshrrev_b16_e64 v4, 8, s1
-; GFX8-NEXT: s_bfe_i32 s5, s1, 0x80010
-; GFX8-NEXT: s_ashr_i32 s4, s1, 24
-; GFX8-NEXT: s_sext_i32_i8 s1, s1
-; GFX8-NEXT: s_ashr_i32 s2, s0, 24
-; GFX8-NEXT: s_bfe_i32 s3, s0, 0x80010
-; GFX8-NEXT: s_sext_i32_i8 s0, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_ushort v1, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v7, 8, v4
; GFX8-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX8-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-NEXT: v_ashrrev_i16_e32 v9, 8, v5
+; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, 8, v0
+; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v2, s0, v5, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, v3, v4, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s3, v6, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: v_ashrrev_i16_e32 v10, 8, v6
+; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
+; GFX8-NEXT: v_mad_u16 v0, v5, v6, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: idot4_acc16_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v5, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v4, 8, s5
-; GFX9-NODL-NEXT: s_bfe_i32 s5, s5, 0x80000
-; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v3, 8, s4
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, s5, v5
-; GFX9-NODL-NEXT: s_bfe_i32 s4, s4, 0x80000
-; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v4, 16, v6
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, s4, v5
-; GFX9-NODL-NEXT: v_lshl_or_b32 v3, v3, 16, v6
-; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v2, 8, s1
-; GFX9-NODL-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v1, 8, s0
-; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX9-NODL-NEXT: v_and_b32_e32 v4, s1, v5
-; GFX9-NODL-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v5
-; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v7, 8, v1
+; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v8, 8, v2
+; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
+; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
+; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v10, 8, v6
+; GFX9-NODL-NEXT: v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
-; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v9, 8, v5
+; GFX9-NODL-NEXT: v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, v1, v2
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_add_u16_e32 v3, v1, v3
+; GFX9-NODL-NEXT: v_lshl_or_b32 v5, v10, 16, v6
+; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v9, 16, v4
+; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
+; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -979,84 +1074,86 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s5
-; GFX9-DL-NEXT: s_bfe_i32 s5, s5, 0x80000
-; GFX9-DL-NEXT: v_ashrrev_i16_e64 v3, 8, s4
-; GFX9-DL-NEXT: v_and_b32_e32 v6, s5, v5
-; GFX9-DL-NEXT: s_bfe_i32 s4, s4, 0x80000
-; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v6
-; GFX9-DL-NEXT: v_and_b32_e32 v6, s4, v5
-; GFX9-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6
-; GFX9-DL-NEXT: v_ashrrev_i16_e64 v2, 8, s1
-; GFX9-DL-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX9-DL-NEXT: v_ashrrev_i16_e64 v1, 8, s0
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX9-DL-NEXT: v_and_b32_e32 v4, s1, v5
-; GFX9-DL-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX9-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v5
-; GFX9-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 8, v1
+; GFX9-DL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 8, v2
+; GFX9-DL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
+; GFX9-DL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 8, v6
+; GFX9-DL-NEXT: v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
-; GFX9-DL-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 8, v5
+; GFX9-DL-NEXT: v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v2, v1, v2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v3, v1, v3
+; GFX9-DL-NEXT: v_lshl_or_b32 v5, v10, 16, v6
+; GFX9-DL-NEXT: v_lshl_or_b32 v4, v9, 16, v4
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
+; GFX9-DL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_bfe_i32 s1, s6, 0x80000
-; GFX10-DL-NEXT: s_bfe_i32 s2, s7, 0x80000
-; GFX10-DL-NEXT: v_and_b32_e32 v6, s1, v2
-; GFX10-DL-NEXT: v_ashrrev_i16_e64 v3, 8, s6
-; GFX10-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s7
-; GFX10-DL-NEXT: v_and_b32_e32 v5, s2, v2
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 16
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 16
-; GFX10-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6
-; GFX10-DL-NEXT: v_ashrrev_i16_e64 v7, 8, s0
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
-; GFX10-DL-NEXT: s_bfe_i32 s2, s1, 0x80000
-; GFX10-DL-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX10-DL-NEXT: v_and_b32_e32 v6, s2, v2
-; GFX10-DL-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, s1
-; GFX10-DL-NEXT: v_lshl_or_b32 v2, v7, 16, v2
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v6
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, v1
+; GFX10-DL-NEXT: v_and_b32_sdwa v8, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_ashrrev_i16_e64 v6, 8, v2
+; GFX10-DL-NEXT: v_and_b32_sdwa v7, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v8
+; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
+; GFX10-DL-NEXT: v_ashrrev_i16_e64 v7, 8, v1
+; GFX10-DL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_ashrrev_i16_e64 v8, 8, v2
+; GFX10-DL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v6
+; GFX10-DL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
+; GFX10-DL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v4, v3
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v5
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = sext <4 x i8> %vec1 to <4 x i16>
%cvec2 = sext <4 x i8> %vec2 to <4 x i16>
@@ -1076,3 +1173,5 @@ entry:
store i16 %add4, i16 addrspace(1)* %dst, align 4
ret void
}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index 48f2f5c6a834..4ac7209177a9 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -11,31 +11,34 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, s4
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -43,29 +46,33 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s5, s3, s2
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010
-; GFX8-NEXT: v_mad_u32_u24 v0, s6, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_lshr_b32 s4, s4, 24
-; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 24
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
+; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s3
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
+; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -75,30 +82,20 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v1, v2
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -106,38 +103,40 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc32:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot4_u32_u8 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32
@@ -179,30 +178,33 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s7, s4, s8
-; GFX7-NEXT: s_and_b32 s6, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -210,99 +212,143 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s1, s0
-; GFX8-NEXT: s_and_b32 s0, s2, s0
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
-; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
-; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
-; GFX8-NEXT: s_lshr_b32 s2, s2, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
+; GFX8-NEXT: v_mov_b32_e32 v5, s0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX8-NEXT: v_and_b32_e32 v6, s0, v4
+; GFX8-NEXT: v_and_b32_e32 v8, s0, v8
+; GFX8-NEXT: v_and_b32_sdwa v10, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v0
+; GFX8-NEXT: v_and_b32_e32 v7, s0, v0
+; GFX8-NEXT: v_and_b32_e32 v9, s0, v9
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v1, v6, v7, v1
+; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mad_u16 v1, v8, v9, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_u16 v1, v10, v5, v1
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot4_acc16:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0
-; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
+; GFX9-NODL-NEXT: v_and_b32_e32 v0, s0, v2
+; GFX9-NODL-NEXT: v_and_b32_e32 v5, s0, v3
+; GFX9-NODL-NEXT: v_and_b32_e32 v6, s0, v6
+; GFX9-NODL-NEXT: v_and_b32_e32 v7, s0, v7
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s7, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
+; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v6, v7, v0
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot4_acc16:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
+; GFX9-DL-NEXT: v_and_b32_e32 v0, s0, v2
+; GFX9-DL-NEXT: v_and_b32_e32 v5, s0, v3
+; GFX9-DL-NEXT: v_and_b32_e32 v6, s0, v6
+; GFX9-DL-NEXT: v_and_b32_e32 v7, s0, v7
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
+; GFX9-DL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_and_b32_sdwa v9, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v6, v7, v0
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc16:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX10-DL-NEXT: v_and_b32_e32 v4, s0, v1
+; GFX10-DL-NEXT: v_and_b32_e32 v10, s0, v2
+; GFX10-DL-NEXT: v_and_b32_e32 v5, s0, v5
+; GFX10-DL-NEXT: v_and_b32_e32 v6, s0, v6
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
+; GFX10-DL-NEXT: v_and_b32_sdwa v4, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_mad_u16 v3, v5, v6, v3
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
+; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i16
@@ -344,30 +390,33 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s7, s4, s8
-; GFX7-NEXT: s_and_b32 s6, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -375,99 +424,121 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX8-NEXT: s_and_b32 s3, s2, s0
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x80008
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_bfe_u32 s6, s2, 0x80010
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX8-NEXT: s_lshr_b32 s2, s2, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
+; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot4_acc8:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s8, 0x80010
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot4_acc8:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc8:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v4, v2, v3, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0
@@ -502,22 +573,27 @@ define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s7, s4, s8
-; GFX7-NEXT: s_and_b32 s6, s5, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_bfe_u32 s5, s5, 0x80008
-; GFX7-NEXT: s_bfe_u32 s4, s4, 0x80008
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: buffer_load_ubyte v4, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
+; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -525,97 +601,97 @@ define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s2, s0
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x80008
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v0
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
+; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot2_8:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0
-; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008
-; GFX9-NODL-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot2_8:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s8, s0
-; GFX9-DL-NEXT: s_and_b32 s0, s1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT: s_bfe_u32 s5, s8, 0x80008
-; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot2_8:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s1, s7, s0
-; GFX10-DL-NEXT: s_and_b32 s0, s6, s0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x80008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x80008
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v2, v2, v3, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v2
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0
@@ -638,30 +714,33 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %sr
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v1, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -669,99 +748,121 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %sr
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s1, s0
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT: s_and_b32 s0, s2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: s_lshr_b32 s2, s2, 24
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v4, v5
+; GFX8-NEXT: v_mad_u16 v0, v8, v7, v0
+; GFX8-NEXT: v_mad_u16 v0, v6, v1, v0
+; GFX8-NEXT: v_mad_u16 v0, v10, v9, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v3, v2, v4
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v7, v6, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v5, v0, v2
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v9, v8, v0
+; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot4_CommutationInsideMAD:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s1, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v3, v2, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v6, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v5, v0, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v8, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_CommutationInsideMAD:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s7, s6, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_mad_u16 v4, v3, v2, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v5, v11, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v7, v6, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0
@@ -796,30 +897,33 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: v_mov_b32_e32 v2, s6
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v3, v6, v3, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v1, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -827,127 +931,122 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT: s_and_b32 s3, s1, s0
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
-; GFX8-NEXT: s_and_b32 s0, s2, s0
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: s_lshr_b32 s2, s2, 24
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ubyte v10, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mad_u16 v6, v7, v6, v10
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v4, v6
+; GFX8-NEXT: v_mad_u16 v0, v5, v1, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v8, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ubyte v9, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v5, v6, v5, v9
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v3, v2, v5
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v4, v0, v2
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v7, v0
+; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX9-DL-NEXT: s_and_b32 s4, s1, s0
-; GFX9-DL-NEXT: s_bfe_u32 s6, s8, 0x80008
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-DL-NEXT: s_and_b32 s0, s8, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT: s_bfe_u32 s9, s8, 0x80010
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-DL-NEXT: s_lshr_b32 s8, s8, 24
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v9, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v5, v6, v5, v9
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v3, v2, v5
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v4, v0, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v7, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_movk_i32 s2, 0xff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80008
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
-; GFX10-DL-NEXT: s_and_b32 s0, s6, s2
-; GFX10-DL-NEXT: s_and_b32 s1, s7, s2
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80010
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v0, v5, v7, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v5, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0
@@ -980,32 +1079,35 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT: v_mad_u32_u24 v1, s6, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s9
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v8, v1, v5, s4
+; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v8
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1013,30 +1115,34 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
+; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s5, s3, s2
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008
-; GFX8-NEXT: v_mad_u32_u24 v1, s5, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010
-; GFX8-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_lshr_b32 s4, s4, 24
-; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 24
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v8, v1, v2, s3
+; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v8
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
+; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1046,31 +1152,25 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v3, v4
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s4, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s6, v3, v2
-; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v1, v2
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
+; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v3, v4, s1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1078,67 +1178,62 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v2
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v4
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s4, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v3, v2
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
+; GFX9-DL-NEXT: v_mad_u32_u24 v3, v3, v4, s1
+; GFX9-DL-NEXT: v_add3_u32 v2, v5, v3, v2
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_multiuse_mul1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_movk_i32 s3, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_movk_i32 s4, 0xff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_and_b32_e32 v3, s3, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v0, v3
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_and_b32 s5, s0, s4
-; GFX10-DL-NEXT: s_and_b32 s4, s1, s4
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s7, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
-; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010
-; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v3, s2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v7, v0, v5
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32
@@ -1180,33 +1275,36 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, s8
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: s_and_b32 s7, s5, s8
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v0, s9
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: v_mov_b32_e32 v2, s7
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s12, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v2, s11
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v2, v0
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v2, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v2, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, s5
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_add_i32_e32 v6, vcc, s5, v3
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v6
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1214,31 +1312,35 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
+; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
+; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s5, s3, s2
-; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: v_mad_u32_u24 v0, s6, v0, v1
-; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s10, v0
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s9
-; GFX8-NEXT: s_lshr_b32 s4, s4, 24
-; GFX8-NEXT: v_mad_u32_u24 v0, s8, v2, v0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 24
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_mad_u32_u24 v0, s3, v2, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, s3
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s3, v4
+; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v5
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1248,32 +1350,24 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 8
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_bfe_u32 v5, v2, 8, 8
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v1, v2
-; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, s10, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v3, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v3, v1
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v2
+; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v4, v5, s0
+; GFX9-NODL-NEXT: v_add_u32_e32 v4, s0, v2
+; GFX9-NODL-NEXT: v_add3_u32 v2, v2, v3, v6
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v1, v4
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1281,69 +1375,60 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_u32 v4, v1, 8, 8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 8, 8
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v1, v2
-; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
-; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-DL-NEXT: v_add_u32_e32 v2, s10, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v3, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v3, v1
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_mad_u32_u24 v2, v4, v5, s0
+; GFX9-DL-NEXT: v_add_u32_e32 v4, s0, v2
+; GFX9-DL-NEXT: v_add3_u32 v2, v2, v3, v6
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, v1, v4
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_multiuse_add1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_movk_i32 s6, 0xff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 8, 8
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80008
-; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: s_and_b32 s4, s0, s6
-; GFX10-DL-NEXT: s_and_b32 s5, s1, s6
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s4, s5, v0
-; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010
-; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s4, s5, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, s8, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
-; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v6, v0, v3, s2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, s2, v6
+; GFX10-DL-NEXT: v_add3_u32 v0, v6, v4, v3
+; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32
@@ -1386,32 +1471,35 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sext_i32_i8 s6, s4
-; GFX7-NEXT: s_sext_i32_i8 s7, s5
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80008
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT: s_and_b32 s6, s6, s8
-; GFX7-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x80010
-; GFX7-NEXT: s_lshr_b32 s5, s5, 24
-; GFX7-NEXT: v_mov_b32_e32 v2, s11
-; GFX7-NEXT: s_lshr_b32 s4, s4, 24
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1419,123 +1507,143 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_movk_i32 s2, 0xff
+; GFX8-NEXT: v_mov_b32_e32 v5, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x80008
-; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008
-; GFX8-NEXT: s_sext_i32_i8 s3, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX8-NEXT: s_sext_i32_i8 s2, s0
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x80010
-; GFX8-NEXT: s_lshr_b32 s1, s1, 24
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: s_lshr_b32 s0, s0, 24
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v3, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ushort v10, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
+; GFX8-NEXT: v_and_b32_e32 v7, s2, v7
+; GFX8-NEXT: v_bfe_i32 v1, v4, 0, 8
+; GFX8-NEXT: v_and_b32_sdwa v9, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX8-NEXT: v_and_b32_e32 v8, s2, v8
+; GFX8-NEXT: v_bfe_i32 v6, v0, 0, 8
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mad_u16 v7, v7, v8, v10
+; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mad_u16 v1, v1, v6, v7
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT: v_mad_u16 v1, v9, v5, v1
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: notdot4_mixedtypes:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9
-; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NODL-NEXT: global_load_ushort v9, v1, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
+; GFX9-NODL-NEXT: v_and_b32_e32 v5, s0, v5
+; GFX9-NODL-NEXT: v_and_b32_e32 v6, s0, v6
+; GFX9-NODL-NEXT: v_bfe_i32 v0, v2, 0, 8
+; GFX9-NODL-NEXT: v_bfe_i32 v4, v3, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v3, v1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v4, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v5, v5, v6, v9
+; GFX9-NODL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v7, v8, v0
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: notdot4_mixedtypes:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008
-; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008
-; GFX9-DL-NEXT: s_sext_i32_i8 s5, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010
-; GFX9-DL-NEXT: s_sext_i32_i8 s4, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010
-; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s9
-; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ushort v9, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
+; GFX9-DL-NEXT: v_and_b32_e32 v5, s0, v5
+; GFX9-DL-NEXT: v_and_b32_e32 v6, s0, v6
+; GFX9-DL-NEXT: v_bfe_i32 v0, v2, 0, 8
+; GFX9-DL-NEXT: v_bfe_i32 v4, v3, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v3, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v4, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v5, v5, v6, v9
+; GFX9-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_and_b32_sdwa v8, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v7, v8, v0
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: notdot4_mixedtypes:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80008
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v9, v2, 0, 8
+; GFX10-DL-NEXT: v_and_b32_e32 v4, s0, v4
+; GFX10-DL-NEXT: v_and_b32_e32 v5, s0, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_sext_i32_i8 s0, s6
-; GFX10-DL-NEXT: s_sext_i32_i8 s1, s7
-; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80010
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX10-DL-NEXT: v_and_b32_sdwa v4, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_mad_u16 v3, v6, v9, v3
+; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i16
@@ -1577,32 +1685,34 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_movk_i32 s11, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 24
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010
-; GFX7-NEXT: s_lshr_b32 s8, s5, 24
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010
-; GFX7-NEXT: s_and_b32 s5, s5, s11
-; GFX7-NEXT: s_and_b32 s4, s4, s11
-; GFX7-NEXT: s_load_dword s11, s[0:1], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v7, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1610,29 +1720,33 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s5, s3, 24
-; GFX8-NEXT: s_lshr_b32 s6, s4, 24
-; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80010
-; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s3
-; GFX8-NEXT: s_and_b32 s3, s3, s2
-; GFX8-NEXT: s_and_b32 s2, s4, s2
-; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80010
-; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s4
-; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 8
+; GFX8-NEXT: v_lshrrev_b16_e32 v5, 8, v3
+; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v0
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 8
+; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v2, v3
-; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s3
+; GFX8-NEXT: v_mad_u32_u24 v0, v5, v7, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v4, v6, v0
+; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1642,30 +1756,20 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 24
-; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80010
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-NODL-NEXT: s_and_b32 s0, s0, s8
-; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-NODL-NEXT: s_and_b32 s1, s1, s8
-; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-NODL-NEXT: v_mad_u32_u24 v3, s0, v3, v4
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v1, v2, v3
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1673,68 +1777,56 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_movk_i32 s8, 0xff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 24
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80010
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-DL-NEXT: s_and_b32 s0, s0, s8
-; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80010
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-DL-NEXT: s_and_b32 s1, s1, s8
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-DL-NEXT: v_mad_u32_u24 v3, s0, v3, v4
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, v3
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
+; GFX9-DL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-DL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_mov_b32 s3, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_movk_i32 s5, 0xff
-; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_sdwa v0, s3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_and_b32_sdwa v3, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v0, v0, v3
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-DL-NEXT: s_and_b32 s6, s0, s5
-; GFX10-DL-NEXT: s_and_b32 s5, s1, s5
-; GFX10-DL-NEXT: v_and_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-DL-NEXT: v_and_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-DL-NEXT: v_mad_u32_u24 v2, s6, s5, v2
-; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010
-; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v1, v2
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: v_add3_u32 v0, v7, s2, v0
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
+; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = zext <4 x i8> %vec1 to <4 x i32>
%cvec2 = zext <4 x i8> %vec2 to <4 x i32>
@@ -1762,30 +1854,42 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 24
-; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010
-; GFX7-NEXT: s_lshr_b32 s9, s5, 24
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT: s_and_b32 s4, s4, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: v_mov_b32_e32 v3, s12
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xff00
+; GFX7-NEXT: s_movk_i32 s5, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
+; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v5
+; GFX7-NEXT: v_and_b32_e32 v6, s5, v0
+; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v1, s5, v1
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v8
+; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1793,65 +1897,74 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_movk_i32 s2, 0xff
+; GFX8-NEXT: v_mov_b32_e32 v5, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s6, s1, s0
-; GFX8-NEXT: s_and_b32 s0, s2, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
-; GFX8-NEXT: v_lshrrev_b16_e64 v3, 8, s2
-; GFX8-NEXT: v_lshrrev_b16_e64 v4, 8, s1
-; GFX8-NEXT: s_lshr_b32 s4, s2, 24
-; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80010
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: s_lshr_b32 s3, s1, 24
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ushort v10, v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_and_b32_sdwa v9, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v4
+; GFX8-NEXT: v_lshrrev_b16_e32 v6, 8, v4
+; GFX8-NEXT: v_and_b32_e32 v4, s2, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v0
+; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, v4, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v6, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v10
+; GFX8-NEXT: v_mad_u16 v0, v6, v8, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v5, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v7, v0
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: udot4_acc16_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s0, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s7, s1, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-NODL-NEXT: v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NODL-NEXT: s_lshr_b32 s6, s1, 24
-; GFX9-NODL-NEXT: v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_lshl_or_b32 v4, s6, 16, v4
-; GFX9-NODL-NEXT: v_lshl_or_b32 v5, s4, 16, v5
-; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v4, v5, v4
-; GFX9-NODL-NEXT: v_and_b32_sdwa v5, v3, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-NODL-NEXT: v_and_b32_sdwa v3, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v5, 8, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v7, 16, v2
+; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX9-NODL-NEXT: v_and_b32_e32 v10, v4, v10
+; GFX9-NODL-NEXT: v_and_b32_e32 v4, v4, v9
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
-; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, v1, v2
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_add_u16_e32 v3, v1, v3
+; GFX9-NODL-NEXT: v_lshl_or_b32 v5, v8, 16, v10
+; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
+; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
+; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -1859,76 +1972,88 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s5, s0, 16
-; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 16
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-DL-NEXT: v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 24
-; GFX9-DL-NEXT: v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_lshl_or_b32 v4, s6, 16, v4
-; GFX9-DL-NEXT: v_lshl_or_b32 v5, s4, 16, v5
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v4
-; GFX9-DL-NEXT: v_and_b32_sdwa v5, v3, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1
-; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0
-; GFX9-DL-NEXT: v_and_b32_sdwa v3, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v5, 8, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-DL-NEXT: v_and_b32_sdwa v9, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX9-DL-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_lshl_or_b32 v2, v7, 16, v2
+; GFX9-DL-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX9-DL-NEXT: v_and_b32_e32 v10, v4, v10
+; GFX9-DL-NEXT: v_and_b32_e32 v4, v4, v9
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
-; GFX9-DL-NEXT: global_load_ushort v2, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v2, v1, v2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v3, v1, v3
+; GFX9-DL-NEXT: v_lshl_or_b32 v5, v8, 16, v10
+; GFX9-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
+; GFX9-DL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
+; GFX10-DL-NEXT: s_movk_i32 s2, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s6
-; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s7
-; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: s_lshr_b32 s0, s7, 16
-; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 16
-; GFX10-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
-; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 24
-; GFX10-DL-NEXT: s_lshr_b32 s0, s7, 24
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, s0, 16, v5
-; GFX10-DL-NEXT: v_lshl_or_b32 v2, s1, 16, v2
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, v1
+; GFX10-DL-NEXT: v_and_b32_sdwa v8, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v6, 8, v2
+; GFX10-DL-NEXT: v_and_b32_sdwa v7, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_and_b32_sdwa v9, v1, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_and_b32_sdwa v10, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v8
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v10
+; GFX10-DL-NEXT: v_and_b32_e32 v4, v4, v9
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v5, v5, v6
+; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v7
+; GFX10-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v5, v3
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = zext <4 x i8> %vec1 to <4 x i16>
%cvec2 = zext <4 x i8> %vec2 to <4 x i16>
@@ -1956,43 +2081,33 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_u32 s6, s4, 0x80008
-; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT: s_lshr_b32 s11, s5, 16
-; GFX7-NEXT: s_lshr_b32 s12, s5, 24
-; GFX7-NEXT: v_mov_b32_e32 v3, s10
-; GFX7-NEXT: s_lshr_b32 s7, s4, 16
-; GFX7-NEXT: v_mov_b32_e32 v2, s11
-; GFX7-NEXT: s_lshr_b32 s9, s4, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: s_mul_i32 s4, s4, s5
-; GFX7-NEXT: v_mul_u32_u24_e32 v1, s9, v1
-; GFX7-NEXT: v_mul_u32_u24_e32 v2, s7, v2
-; GFX7-NEXT: v_mul_u32_u24_e32 v3, s6, v3
-; GFX7-NEXT: s_and_b32 s5, s4, s8
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_or_b32_e32 v2, s5, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 8, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_and_b32_e32 v3, s4, v2
+; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v0
+; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v3, v4, v7, v3
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v3
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2000,39 +2115,36 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s0, 24
-; GFX8-NEXT: s_lshr_b32 s4, s1, 24
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: s_mul_i32 s0, s0, s1
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v7, s2
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX8-NEXT: v_mul_u32_u24_e32 v5, s3, v5
-; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX8-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v4, v3, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_mul_lo_u16_e32 v9, v5, v6
+; GFX8-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v8
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v7
+; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v7
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -2040,34 +2152,29 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT: global_load_ubyte v4, v0, s[2:3]
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT: s_lshr_b32 s6, s1, 16
-; GFX9-NODL-NEXT: s_lshr_b32 s7, s1, 24
-; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v1, s0, v1
-; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v2, s0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NODL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT: s_lshr_b32 s5, s0, 24
-; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v3, s4, v3
-; GFX9-NODL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NODL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NODL-NEXT: v_or_b32_e32 v2, v1, v2
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v8, v4, v5
+; GFX9-NODL-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v7, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX9-NODL-NEXT: v_or_b32_e32 v7, v7, v8
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v7
+; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
+; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -2075,76 +2182,76 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ubyte v4, v0, s[2:3]
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 16
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 16
-; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 24
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v1, s0, v1
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, s0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-DL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-DL-NEXT: s_lshr_b32 s5, s0, 24
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, s4, v3
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-DL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v2, v1, v2
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, v4, v5
+; GFX9-DL-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX9-DL-NEXT: v_or_b32_e32 v7, v7, v8
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v7
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot4_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s6
-; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s7
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s0, s1
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, v2, v3
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v3, s6, s7
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 16
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 16
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v2, 8, v2
-; GFX10-DL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v3, 8, v4
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s0, s1
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-DL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NEXT: v_or_b32_e32 v3, v2, v3
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v3
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[0:1]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v8, 8, v2
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, v4, v5
+; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, v1
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v6, v7
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v4
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v3, v1, v2, v3
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, v5, v8
+; GFX10-DL-NEXT: v_or_b32_sdwa v4, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v5, 8, v5
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v4
+; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v3, v5
+; GFX10-DL-NEXT: v_mad_u16 v1, v6, v7, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v2
+; GFX10-DL-NEXT: global_store_byte v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
- %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
+ %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%mul = mul <4 x i8> %vec1, %vec2
%mul0 = extractelement <4 x i8> %mul, i64 0
@@ -2161,3 +2268,5 @@ entry:
store i8 %add4, i8 addrspace(1)* %dst, align 4
ret void
}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 949ade9d6c90..e076a602a9da 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -11,105 +11,104 @@
define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc32:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
-; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
-; GFX7-NEXT: s_bfe_i32 s9, s5, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x40008
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_bfe_i32 s13, s5, 0x4000c
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40010
-; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
-; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40010
-; GFX7-NEXT: v_mov_b32_e32 v1, s15
-; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40014
-; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40018
-; GFX7-NEXT: v_mad_i32_i24 v0, s14, v1, v0
-; GFX7-NEXT: s_bfe_i32 s16, s4, 0x40014
-; GFX7-NEXT: v_mov_b32_e32 v1, s17
-; GFX7-NEXT: s_bfe_i32 s18, s4, 0x40018
-; GFX7-NEXT: v_mad_i32_i24 v0, s16, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: s_ashr_i32 s5, s5, 28
-; GFX7-NEXT: v_mad_i32_i24 v0, s18, v1, v0
-; GFX7-NEXT: s_ashr_i32 s4, s4, 28
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 4
+; GFX7-NEXT: v_bfe_i32 v3, v2, 4, 4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v9, v0, 0, 4
+; GFX7-NEXT: v_bfe_i32 v10, v0, 4, 4
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v9, s4
+; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 8, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v3, v10, v1
+; GFX7-NEXT: v_bfe_i32 v5, v2, 12, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 12, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v4, v11, v1
+; GFX7-NEXT: v_bfe_i32 v6, v2, 16, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 16, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v5, v12, v1
+; GFX7-NEXT: v_bfe_i32 v7, v2, 20, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v6, v13, v1
+; GFX7-NEXT: v_bfe_i32 v8, v2, 24, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 24, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v7, v14, v1
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX7-NEXT: v_mad_i32_i24 v1, v8, v15, v1
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot8_acc32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_i32 s4, s2, 0x40000
-; GFX8-NEXT: s_bfe_i32 s5, s3, 0x40000
-; GFX8-NEXT: s_bfe_i32 s7, s3, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_bfe_i32 s9, s3, 0x40008
-; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_bfe_i32 s11, s3, 0x4000c
-; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40010
-; GFX8-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: s_bfe_i32 s15, s3, 0x40014
-; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40018
-; GFX8-NEXT: v_mad_i32_i24 v0, s12, v1, v0
-; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT: v_mov_b32_e32 v1, s15
-; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT: v_mad_i32_i24 v0, s14, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: s_ashr_i32 s3, s3, 28
-; GFX8-NEXT: v_mad_i32_i24 v0, s16, v1, v0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 28
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 4
+; GFX8-NEXT: v_bfe_i32 v4, v3, 4, 4
+; GFX8-NEXT: v_bfe_i32 v6, v3, 8, 4
+; GFX8-NEXT: v_bfe_i32 v8, v3, 12, 4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 4
+; GFX8-NEXT: v_bfe_i32 v5, v0, 4, 4
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s2
+; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
+; GFX8-NEXT: v_bfe_i32 v9, v0, 12, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
+; GFX8-NEXT: v_bfe_i32 v10, v3, 16, 4
+; GFX8-NEXT: v_bfe_i32 v11, v0, 16, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v8, v9, v1
+; GFX8-NEXT: v_bfe_i32 v12, v3, 20, 4
+; GFX8-NEXT: v_bfe_i32 v13, v0, 20, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v10, v11, v1
+; GFX8-NEXT: v_bfe_i32 v14, v3, 24, 4
+; GFX8-NEXT: v_bfe_i32 v15, v0, 24, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v12, v13, v1
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 28, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX8-NEXT: v_mad_i32_i24 v1, v14, v15, v1
+; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -117,120 +116,117 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: idot8_acc32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_bfe_i32 v3, v1, 0, 4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_bfe_i32 v4, v2, 0, 4
+; GFX9-NEXT: v_bfe_i32 v5, v1, 4, 4
+; GFX9-NEXT: v_bfe_i32 v6, v2, 4, 4
+; GFX9-NEXT: v_mul_i32_i24_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_i32 v7, v1, 8, 4
+; GFX9-NEXT: v_bfe_i32 v8, v2, 8, 4
+; GFX9-NEXT: v_mul_i32_i24_e32 v4, v5, v6
+; GFX9-NEXT: v_bfe_i32 v9, v1, 12, 4
+; GFX9-NEXT: v_bfe_i32 v10, v2, 12, 4
+; GFX9-NEXT: v_bfe_i32 v11, v1, 16, 4
+; GFX9-NEXT: v_bfe_i32 v12, v2, 16, 4
+; GFX9-NEXT: v_bfe_i32 v13, v1, 20, 4
+; GFX9-NEXT: v_bfe_i32 v15, v1, 24, 4
+; GFX9-NEXT: v_bfe_i32 v14, v2, 20, 4
+; GFX9-NEXT: v_bfe_i32 v16, v2, 24, 4
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 28, v1
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX9-NEXT: v_mul_i32_i24_e32 v5, v7, v8
+; GFX9-NEXT: v_mul_i32_i24_e32 v6, v9, v10
+; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s4, s0, 0x40000
-; GFX9-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40008
-; GFX9-NEXT: v_mad_i32_i24 v1, s6, v2, v1
-; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-NEXT: s_bfe_i32 s11, s1, 0x4000c
-; GFX9-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX9-NEXT: s_bfe_i32 s10, s0, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-NEXT: v_mad_i32_i24 v1, s10, v2, v1
-; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010
-; GFX9-NEXT: v_mov_b32_e32 v2, s13
-; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-NEXT: v_mad_i32_i24 v1, s12, v2, v1
-; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014
-; GFX9-NEXT: v_mov_b32_e32 v2, s15
-; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018
-; GFX9-NEXT: v_mad_i32_i24 v1, s14, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s17
-; GFX9-NEXT: s_ashr_i32 s1, s1, 28
-; GFX9-NEXT: v_mad_i32_i24 v1, s16, v2, v1
-; GFX9-NEXT: s_ashr_i32 s0, s0, 28
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NEXT: v_mul_i32_i24_e32 v7, v11, v12
+; GFX9-NEXT: v_mul_i32_i24_e32 v8, v13, v14
+; GFX9-NEXT: v_add3_u32 v2, v2, v5, v6
+; GFX9-NEXT: v_mul_i32_i24_e32 v9, v15, v16
+; GFX9-NEXT: v_add3_u32 v2, v2, v7, v8
+; GFX9-NEXT: v_add3_u32 v1, v2, v9, v1
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc32:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s14, -1
-; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s12, s12, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot8_i32_i4 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot8_i32_i4 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc32:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_clause 0x1
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
-; GFX10-DL-XNACK-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v1, v1, v2, s2
+; GFX10-DL-XNACK-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc32:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
-; GFX10-DL-NOXNACK-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, v1, v0, s2
+; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc32:
; GFX10-DL: ; %bb.0: ; %entry
@@ -256,8 +252,11 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i32
@@ -326,71 +325,71 @@ entry:
define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc16:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
-; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
-; GFX7-NEXT: s_bfe_i32 s10, s5, 0x40004
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40004
-; GFX7-NEXT: s_bfe_i32 s12, s5, 0x40008
-; GFX7-NEXT: s_and_b32 s10, s10, s8
-; GFX7-NEXT: s_and_b32 s6, s6, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_i32 s14, s5, 0x4000c
-; GFX7-NEXT: s_and_b32 s12, s12, s8
-; GFX7-NEXT: s_and_b32 s9, s9, s8
-; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: s_bfe_i32 s13, s4, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
-; GFX7-NEXT: s_and_b32 s14, s14, s8
-; GFX7-NEXT: s_and_b32 s11, s11, s8
-; GFX7-NEXT: v_mov_b32_e32 v3, s12
-; GFX7-NEXT: s_bfe_i32 s15, s4, 0x40010
-; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40014
-; GFX7-NEXT: s_and_b32 s16, s16, s8
-; GFX7-NEXT: s_and_b32 s13, s13, s8
-; GFX7-NEXT: v_mov_b32_e32 v4, s14
-; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40018
-; GFX7-NEXT: s_bfe_i32 s17, s4, 0x40014
-; GFX7-NEXT: s_and_b32 s18, s18, s8
-; GFX7-NEXT: s_and_b32 s15, s15, s8
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: s_bfe_i32 s19, s4, 0x40018
-; GFX7-NEXT: s_ashr_i32 s5, s5, 28
-; GFX7-NEXT: s_and_b32 s20, s20, s8
-; GFX7-NEXT: s_and_b32 s17, s17, s8
-; GFX7-NEXT: v_mov_b32_e32 v6, s18
-; GFX7-NEXT: s_ashr_i32 s4, s4, 28
-; GFX7-NEXT: s_and_b32 s19, s19, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: s_and_b32 s4, s4, s8
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v3, 0, 4
+; GFX7-NEXT: v_bfe_i32 v4, v3, 4, 4
+; GFX7-NEXT: v_bfe_i32 v5, v3, 8, 4
+; GFX7-NEXT: v_bfe_i32 v6, v3, 12, 4
+; GFX7-NEXT: v_bfe_i32 v7, v3, 16, 4
+; GFX7-NEXT: v_bfe_i32 v8, v3, 20, 4
+; GFX7-NEXT: v_bfe_i32 v9, v3, 24, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s13, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s15, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s17, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s19, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_bfe_i32 v10, v0, 0, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 4, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 8, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 16, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 20, 4
+; GFX7-NEXT: v_bfe_i32 v16, v0, 24, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
+; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX7-NEXT: v_and_b32_e32 v16, v2, v16
+; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: v_and_b32_e32 v10, s4, v10
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_and_b32_e32 v11, s4, v11
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT: v_and_b32_e32 v12, s4, v12
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
+; GFX7-NEXT: v_and_b32_e32 v13, s4, v13
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT: v_and_b32_e32 v14, s4, v14
+; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v10, v2
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v11, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v12, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v6, v13, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v7, v14, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v8, v15, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v9, v16, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -398,273 +397,381 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, 12
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_i32 s4, s0, 0x40000
-; GFX8-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX8-NEXT: s_bfe_i32 s9, s1, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v6, s5
-; GFX8-NEXT: s_lshr_b32 s2, s0, 12
-; GFX8-NEXT: s_lshr_b32 s3, s1, 12
-; GFX8-NEXT: s_bfe_i32 s6, s0, 0x40004
-; GFX8-NEXT: s_bfe_i32 s8, s0, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
-; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s2
-; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s3
-; GFX8-NEXT: v_mul_i32_i24_e32 v3, s8, v3
-; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40010
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT: s_bfe_i32 s13, s1, 0x40014
-; GFX8-NEXT: s_bfe_i32 s10, s0, 0x40010
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40018
-; GFX8-NEXT: s_bfe_i32 s12, s0, 0x40014
-; GFX8-NEXT: v_mov_b32_e32 v9, s13
-; GFX8-NEXT: s_bfe_i32 s14, s0, 0x40018
-; GFX8-NEXT: s_ashr_i32 s1, s1, 28
-; GFX8-NEXT: v_mov_b32_e32 v10, s15
-; GFX8-NEXT: s_ashr_i32 s0, s0, 28
+; GFX8-NEXT: flat_load_ushort v4, v[0:1]
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 28, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 20, v3
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v17, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 20, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
+; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v16
+; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v2, s4, v6, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v7, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT: v_mad_u32_u24 v2, v4, v5, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s10, v8, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s12, v9, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s14, v10, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT: v_mad_u16 v4, v5, v16, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
+; GFX8-NEXT: v_mad_u16 v4, v10, v15, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT: v_mad_u16 v4, v9, v14, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
+; GFX8-NEXT: v_mad_u16 v4, v8, v13, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT: v_mad_u16 v4, v17, v18, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX8-NEXT: v_mad_u16 v4, v7, v12, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
+; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: idot8_acc16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v4, 12
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40000
-; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40000
-; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40004
-; GFX9-NEXT: s_bfe_i32 s11, s1, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-NEXT: s_lshr_b32 s4, s0, 12
-; GFX9-NEXT: s_lshr_b32 s5, s1, 12
-; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40004
-; GFX9-NEXT: s_bfe_i32 s10, s0, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s4
-; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s5
-; GFX9-NEXT: v_mul_i32_i24_e32 v2, s10, v2
-; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014
-; GFX9-NEXT: v_mov_b32_e32 v8, s15
-; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018
-; GFX9-NEXT: s_ashr_i32 s1, s1, 28
-; GFX9-NEXT: v_mov_b32_e32 v9, s17
-; GFX9-NEXT: s_ashr_i32 s0, s0, 28
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v16, 12, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX9-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX9-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v15
+; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_i32_i24 v1, s6, v5, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s8, v6, v1
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT: v_mad_u32_u24 v1, v3, v4, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s12, v7, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s14, v8, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s16, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc16:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 12
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x40000
-; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x40000
-; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 12
-; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 12
-; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s10, s0, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s4
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s5
-; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, s10, v2
-; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_i32 s12, s0, 0x40010
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_i32 s14, s0, 0x40014
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s15
-; GFX9-DL-NEXT: s_bfe_i32 s16, s0, 0x40018
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28
-; GFX9-DL-NEXT: v_mov_b32_e32 v9, s17
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 28
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v16, 12, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v15
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v5, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v6, v1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, v3, v4, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s12, v7, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s14, v8, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s16, v9, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v5
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc16:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s0, s6, 12
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s7, 12
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s2, s6, 0x40000
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s3, s7, 0x40000
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s8, s6, 0x40004
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s9, s6, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s10, s7, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s7, 0x40004
-; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40010
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-XNACK-NEXT: global_load_ushort v3, v0, s[0:1]
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v2
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s8, s0, v1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s0, 0xffff
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40010
-; GFX10-DL-XNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40014
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40014
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40018
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40018
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s6, 28
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s7, 28
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v18, v17, v3
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v10, v16, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v9
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v3, v2, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v7
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v8, v15, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v6, v7, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v5
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v4, v7, v1
+; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc16:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s2, s0, 12
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s1, 12
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s6, s0, 0x40000
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s7, s1, 0x40000
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s2
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s3
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s8, s0, 0x40004
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s9, s0, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s10, s1, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s1, 0x40004
-; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: global_load_ushort v3, v2, s[0:1]
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s6, s7, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s8, s2, v1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s2, 0xffff
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, s2, v3
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010
-; GFX10-DL-NOXNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-NOXNACK-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v18, v17, v3
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v10, v16, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v3, v0, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v15, v9, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v4
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v5, v0
+; GFX10-DL-NOXNACK-NEXT: global_store_short v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc16:
; GFX10-DL: ; %bb.0: ; %entry
@@ -720,8 +827,11 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i16
@@ -789,71 +899,71 @@ entry:
define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
-; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
-; GFX7-NEXT: s_bfe_i32 s10, s5, 0x40004
-; GFX7-NEXT: s_and_b32 s7, s7, s8
-; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40004
-; GFX7-NEXT: s_bfe_i32 s12, s5, 0x40008
-; GFX7-NEXT: s_and_b32 s10, s10, s8
-; GFX7-NEXT: s_and_b32 s6, s6, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_i32 s14, s5, 0x4000c
-; GFX7-NEXT: s_and_b32 s12, s12, s8
-; GFX7-NEXT: s_and_b32 s9, s9, s8
-; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: s_bfe_i32 s13, s4, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
-; GFX7-NEXT: s_and_b32 s14, s14, s8
-; GFX7-NEXT: s_and_b32 s11, s11, s8
-; GFX7-NEXT: v_mov_b32_e32 v3, s12
-; GFX7-NEXT: s_bfe_i32 s15, s4, 0x40010
-; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40014
-; GFX7-NEXT: s_and_b32 s16, s16, s8
-; GFX7-NEXT: s_and_b32 s13, s13, s8
-; GFX7-NEXT: v_mov_b32_e32 v4, s14
-; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40018
-; GFX7-NEXT: s_bfe_i32 s17, s4, 0x40014
-; GFX7-NEXT: s_and_b32 s18, s18, s8
-; GFX7-NEXT: s_and_b32 s15, s15, s8
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: s_bfe_i32 s19, s4, 0x40018
-; GFX7-NEXT: s_ashr_i32 s5, s5, 28
-; GFX7-NEXT: s_and_b32 s20, s20, s8
-; GFX7-NEXT: s_and_b32 s17, s17, s8
-; GFX7-NEXT: v_mov_b32_e32 v6, s18
-; GFX7-NEXT: s_ashr_i32 s4, s4, 28
-; GFX7-NEXT: s_and_b32 s19, s19, s8
-; GFX7-NEXT: s_and_b32 s5, s5, s8
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: s_and_b32 s4, s4, s8
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v3, 0, 4
+; GFX7-NEXT: v_bfe_i32 v4, v3, 4, 4
+; GFX7-NEXT: v_bfe_i32 v5, v3, 8, 4
+; GFX7-NEXT: v_bfe_i32 v6, v3, 12, 4
+; GFX7-NEXT: v_bfe_i32 v7, v3, 16, 4
+; GFX7-NEXT: v_bfe_i32 v8, v3, 20, 4
+; GFX7-NEXT: v_bfe_i32 v9, v3, 24, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s13, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s15, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s17, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s19, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT: v_bfe_i32 v10, v0, 0, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 4, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 8, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 16, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 20, 4
+; GFX7-NEXT: v_bfe_i32 v16, v0, 24, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
+; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX7-NEXT: v_and_b32_e32 v16, v2, v16
+; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX7-NEXT: buffer_load_ubyte v2, off, s[0:3], 0
+; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT: v_and_b32_e32 v10, s4, v10
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_and_b32_e32 v11, s4, v11
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT: v_and_b32_e32 v12, s4, v12
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
+; GFX7-NEXT: v_and_b32_e32 v13, s4, v13
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT: v_and_b32_e32 v14, s4, v14
+; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v10, v2
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v11, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v5, v12, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v6, v13, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v7, v14, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v8, v15, v1
+; GFX7-NEXT: v_mad_u32_u24 v1, v9, v16, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -861,282 +971,381 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, 12
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s3, s1, 12
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40000
-; GFX8-NEXT: s_lshr_b32 s4, s2, 12
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40004
-; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40008
-; GFX8-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s3
-; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s4
-; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX8-NEXT: s_bfe_i32 s9, s1, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v3, s10
-; GFX8-NEXT: v_mov_b32_e32 v7, s8
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT: v_mul_i32_i24_e32 v3, s9, v3
-; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT: v_and_b32_e32 v4, s0, v4
-; GFX8-NEXT: v_and_b32_e32 v5, s0, v5
-; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40010
-; GFX8-NEXT: v_mov_b32_e32 v8, s12
-; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT: s_bfe_i32 s13, s1, 0x40014
-; GFX8-NEXT: v_mov_b32_e32 v9, s14
-; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40018
-; GFX8-NEXT: s_ashr_i32 s2, s2, 28
-; GFX8-NEXT: v_mov_b32_e32 v10, s16
-; GFX8-NEXT: s_ashr_i32 s1, s1, 28
+; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 28, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 20, v3
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v17, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 20, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
+; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v16
+; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v2, s5, v6, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s7, v7, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_mad_u32_u24 v2, v4, v5, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s11, v8, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s13, v9, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s15, v10, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT: v_mad_u16 v4, v5, v16, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
+; GFX8-NEXT: v_mad_u16 v4, v10, v15, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT: v_mad_u16 v4, v9, v14, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
+; GFX8-NEXT: v_mad_u16 v4, v8, v13, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT: v_mad_u16 v4, v17, v18, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX8-NEXT: v_mad_u16 v4, v7, v12, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
+; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: idot8_acc8:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v4, 12
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s1, 12
-; GFX9-NEXT: s_bfe_i32 s7, s8, 0x40000
-; GFX9-NEXT: s_lshr_b32 s5, s8, 12
-; GFX9-NEXT: s_bfe_i32 s10, s8, 0x40004
-; GFX9-NEXT: s_bfe_i32 s12, s8, 0x40008
-; GFX9-NEXT: s_bfe_i32 s6, s1, 0x40000
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s4
-; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s5
-; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40004
-; GFX9-NEXT: s_bfe_i32 s11, s1, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_mov_b32_e32 v6, s10
-; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT: v_mul_i32_i24_e32 v2, s11, v2
-; GFX9-NEXT: s_bfe_i32 s14, s8, 0x40010
-; GFX9-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX9-NEXT: v_and_b32_e32 v4, s0, v4
-; GFX9-NEXT: s_bfe_i32 s16, s8, 0x40014
-; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: s_bfe_i32 s18, s8, 0x40018
-; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-NEXT: v_mov_b32_e32 v8, s16
-; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-NEXT: s_ashr_i32 s8, s8, 28
-; GFX9-NEXT: v_mov_b32_e32 v9, s18
-; GFX9-NEXT: s_ashr_i32 s1, s1, 28
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_load_ubyte v3, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v16, 12, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX9-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX9-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v15
+; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_i32_i24 v1, s6, v5, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s9, v6, v1
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_mad_u32_u24 v1, v3, v4, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s13, v7, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s15, v8, v1
-; GFX9-NEXT: v_mad_i32_i24 v1, s17, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NEXT: v_mad_i32_i24 v1, s1, v2, v1
+; GFX9-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
+; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc8:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v4, 12
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s1, 12
-; GFX9-DL-NEXT: s_bfe_i32 s7, s8, 0x40000
-; GFX9-DL-NEXT: s_lshr_b32 s5, s8, 12
-; GFX9-DL-NEXT: s_bfe_i32 s10, s8, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s12, s8, 0x40008
-; GFX9-DL-NEXT: s_bfe_i32 s6, s1, 0x40000
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s4
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s5
-; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s10
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, s11, v2
-; GFX9-DL-NEXT: s_bfe_i32 s14, s8, 0x40010
-; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v4
-; GFX9-DL-NEXT: s_bfe_i32 s16, s8, 0x40014
-; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-DL-NEXT: s_bfe_i32 s18, s8, 0x40018
-; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s16
-; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-DL-NEXT: s_ashr_i32 s8, s8, 28
-; GFX9-DL-NEXT: v_mov_b32_e32 v9, s18
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v16, 12, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v15
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v5, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s9, v6, v1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, v3, v4, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s13, v7, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s15, v8, v1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s17, v9, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v5
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
+; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc8:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s0, s6, 12
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s7, 12
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s2, s6, 0x40000
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s3, s7, 0x40000
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s8, s6, 0x40004
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s9, s6, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s10, s7, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s7, 0x40004
-; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40010
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v0, s[0:1]
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v2
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s8, s0, v1
-; GFX10-DL-XNACK-NEXT: s_movk_i32 s0, 0xff
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40010
-; GFX10-DL-XNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40014
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40014
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40018
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40018
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s6, 28
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s7, 28
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v18, v17, v3
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v10, v16, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v9
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v3, v2, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v7
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v8, v15, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v6, v7, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v5
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v4, v7, v1
+; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc8:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s2, s0, 12
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s1, 12
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s6, s0, 0x40000
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s7, s1, 0x40000
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s2
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s3
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s8, s0, 0x40004
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s9, s0, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s10, s1, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s1, 0x40004
-; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v3, v2, s[0:1]
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s6, s7, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s8, s2, v1
-; GFX10-DL-NOXNACK-NEXT: s_movk_i32 s2, 0xff
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, s2, v3
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010
-; GFX10-DL-NOXNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-NOXNACK-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v18, v17, v3
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v10, v16, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v3, v0, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v15, v9, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v4
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v5, v0
+; GFX10-DL-NOXNACK-NEXT: global_store_byte v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc8:
; GFX10-DL: ; %bb.0: ; %entry
@@ -1192,8 +1401,11 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i8
@@ -1262,109 +1474,108 @@ entry:
define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_multiuses_mul1:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
-; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_i32_i24 v1, s6, v0, v1
-; GFX7-NEXT: s_bfe_i32 s9, s5, 0x40004
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40004
-; GFX7-NEXT: s_bfe_i32 s11, s5, 0x40008
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s9
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v2, v0
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v2, s11
-; GFX7-NEXT: s_bfe_i32 s13, s5, 0x4000c
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v2, v0
-; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
-; GFX7-NEXT: v_mov_b32_e32 v2, s13
-; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40010
-; GFX7-NEXT: v_mad_i32_i24 v0, s12, v2, v0
-; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40010
-; GFX7-NEXT: v_mov_b32_e32 v2, s15
-; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40014
-; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40018
-; GFX7-NEXT: v_mad_i32_i24 v0, s14, v2, v0
-; GFX7-NEXT: s_bfe_i32 s16, s4, 0x40014
-; GFX7-NEXT: v_mov_b32_e32 v2, s17
-; GFX7-NEXT: s_bfe_i32 s18, s4, 0x40018
-; GFX7-NEXT: v_mad_i32_i24 v0, s16, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: s_ashr_i32 s5, s5, 28
-; GFX7-NEXT: v_mad_i32_i24 v0, s18, v2, v0
-; GFX7-NEXT: s_ashr_i32 s4, s4, 28
-; GFX7-NEXT: v_mov_b32_e32 v2, s5
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v2, v0
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 4
+; GFX7-NEXT: v_bfe_i32 v3, v2, 4, 4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_bfe_i32 v9, v0, 0, 4
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_i32_i24 v16, v1, v9, s4
+; GFX7-NEXT: v_bfe_i32 v10, v0, 4, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v9, v16
+; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 8, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v3, v10, v1
+; GFX7-NEXT: v_bfe_i32 v5, v2, 12, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 12, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v4, v11, v1
+; GFX7-NEXT: v_bfe_i32 v6, v2, 16, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 16, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v5, v12, v1
+; GFX7-NEXT: v_bfe_i32 v7, v2, 20, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v6, v13, v1
+; GFX7-NEXT: v_bfe_i32 v8, v2, 24, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 24, 4
+; GFX7-NEXT: v_mad_i32_i24 v1, v7, v14, v1
+; GFX7-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX7-NEXT: v_mad_i32_i24 v1, v8, v15, v1
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v16, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot8_multiuses_mul1:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 4
+; GFX8-NEXT: v_bfe_i32 v4, v3, 4, 4
+; GFX8-NEXT: v_bfe_i32 v6, v3, 8, 4
+; GFX8-NEXT: v_bfe_i32 v8, v3, 12, 4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_i32 s4, s2, 0x40000
-; GFX8-NEXT: s_bfe_i32 s5, s3, 0x40000
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_i32_i24 v1, s4, v0, v1
-; GFX8-NEXT: s_bfe_i32 s7, s3, 0x40004
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004
-; GFX8-NEXT: s_bfe_i32 s9, s3, 0x40008
-; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: v_mad_i32_i24 v0, s6, v2, v0
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v2, s9
-; GFX8-NEXT: s_bfe_i32 s11, s3, 0x4000c
-; GFX8-NEXT: v_mad_i32_i24 v0, s8, v2, v0
-; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT: v_mov_b32_e32 v2, s11
-; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40010
-; GFX8-NEXT: v_mad_i32_i24 v0, s10, v2, v0
-; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT: v_mov_b32_e32 v2, s13
-; GFX8-NEXT: s_bfe_i32 s15, s3, 0x40014
-; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40018
-; GFX8-NEXT: v_mad_i32_i24 v0, s12, v2, v0
-; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT: v_mov_b32_e32 v2, s15
-; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT: v_mad_i32_i24 v0, s14, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s17
-; GFX8-NEXT: s_ashr_i32 s3, s3, 28
-; GFX8-NEXT: v_mad_i32_i24 v0, s16, v2, v0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 28
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_mad_i32_i24 v0, s2, v2, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT: v_mad_i32_i24 v16, v1, v2, s2
+; GFX8-NEXT: v_bfe_i32 v5, v0, 4, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, v16
+; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
+; GFX8-NEXT: v_bfe_i32 v9, v0, 12, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
+; GFX8-NEXT: v_bfe_i32 v10, v3, 16, 4
+; GFX8-NEXT: v_bfe_i32 v11, v0, 16, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v8, v9, v1
+; GFX8-NEXT: v_bfe_i32 v12, v3, 20, 4
+; GFX8-NEXT: v_bfe_i32 v13, v0, 20, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v10, v11, v1
+; GFX8-NEXT: v_bfe_i32 v14, v3, 24, 4
+; GFX8-NEXT: v_bfe_i32 v15, v0, 24, 4
+; GFX8-NEXT: v_mad_i32_i24 v1, v12, v13, v1
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 28, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX8-NEXT: v_mad_i32_i24 v1, v14, v15, v1
+; GFX8-NEXT: v_mad_i32_i24 v0, v3, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v16, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1372,204 +1583,208 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: idot8_multiuses_mul1:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_bfe_i32 v3, v1, 0, 4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_bfe_i32 v4, v2, 0, 4
+; GFX9-NEXT: v_bfe_i32 v5, v1, 4, 4
+; GFX9-NEXT: v_bfe_i32 v6, v2, 4, 4
+; GFX9-NEXT: v_bfe_i32 v7, v1, 8, 4
+; GFX9-NEXT: v_bfe_i32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_i32 v9, v1, 12, 4
+; GFX9-NEXT: v_bfe_i32 v10, v2, 12, 4
+; GFX9-NEXT: v_bfe_i32 v11, v1, 16, 4
+; GFX9-NEXT: v_bfe_i32 v12, v2, 16, 4
+; GFX9-NEXT: v_bfe_i32 v13, v1, 20, 4
+; GFX9-NEXT: v_bfe_i32 v15, v1, 24, 4
+; GFX9-NEXT: v_bfe_i32 v14, v2, 20, 4
+; GFX9-NEXT: v_bfe_i32 v16, v2, 24, 4
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 28, v1
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s4, s0, 0x40000
-; GFX9-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_i32_i24 v2, s4, v1, v2
-; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40004
-; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40008
-; GFX9-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: v_mad_i32_i24 v1, s6, v3, v1
-; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-NEXT: s_bfe_i32 s11, s1, 0x4000c
-; GFX9-NEXT: v_mad_i32_i24 v1, s8, v3, v1
-; GFX9-NEXT: s_bfe_i32 s10, s0, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-NEXT: v_mad_i32_i24 v1, s10, v3, v1
-; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010
-; GFX9-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-NEXT: v_mad_i32_i24 v1, s12, v3, v1
-; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014
-; GFX9-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018
-; GFX9-NEXT: v_mad_i32_i24 v1, s14, v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: s_ashr_i32 s1, s1, 28
-; GFX9-NEXT: v_mad_i32_i24 v1, s16, v3, v1
-; GFX9-NEXT: s_ashr_i32 s0, s0, 28
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mad_i32_i24 v1, s0, v3, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-NEXT: v_mad_i32_i24 v2, v3, v4, s0
+; GFX9-NEXT: v_mul_i32_i24_e32 v5, v5, v6
+; GFX9-NEXT: v_mul_i32_i24_e32 v6, v7, v8
+; GFX9-NEXT: v_mad_i32_i24 v3, v3, v4, v2
+; GFX9-NEXT: v_mul_i32_i24_e32 v7, v9, v10
+; GFX9-NEXT: v_mul_i32_i24_e32 v8, v11, v12
+; GFX9-NEXT: v_add3_u32 v3, v3, v5, v6
+; GFX9-NEXT: v_mul_i32_i24_e32 v9, v13, v14
+; GFX9-NEXT: v_mul_i32_i24_e32 v10, v15, v16
+; GFX9-NEXT: v_add3_u32 v3, v3, v7, v8
+; GFX9-NEXT: v_add3_u32 v3, v3, v9, v10
+; GFX9-NEXT: v_add3_u32 v1, v3, v1, v2
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_multiuses_mul1:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 4
+; GFX9-DL-NEXT: v_bfe_i32 v5, v1, 4, 4
+; GFX9-DL-NEXT: v_bfe_i32 v6, v2, 4, 4
+; GFX9-DL-NEXT: v_bfe_i32 v7, v1, 8, 4
+; GFX9-DL-NEXT: v_bfe_i32 v8, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_i32 v9, v1, 12, 4
+; GFX9-DL-NEXT: v_bfe_i32 v10, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_i32 v11, v1, 16, 4
+; GFX9-DL-NEXT: v_bfe_i32 v12, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_i32 v13, v1, 20, 4
+; GFX9-DL-NEXT: v_bfe_i32 v15, v1, 24, 4
+; GFX9-DL-NEXT: v_bfe_i32 v14, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_i32 v16, v2, 24, 4
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 28, v1
+; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_i32 s4, s0, 0x40000
-; GFX9-DL-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v1, v2
-; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x40004
-; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40008
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v3, v1
-; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x4000c
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v3, v1
-; GFX9-DL-NEXT: s_bfe_i32 s10, s0, 0x4000c
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s10, v3, v1
-; GFX9-DL-NEXT: s_bfe_i32 s12, s0, 0x40010
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s12, v3, v1
-; GFX9-DL-NEXT: s_bfe_i32 s14, s0, 0x40014
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-DL-NEXT: s_bfe_i32 s16, s0, 0x40018
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s14, v3, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s16, v3, v1
-; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 28
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v3, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX9-DL-NEXT: v_mad_i32_i24 v2, v3, v4, s0
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v5, v5, v6
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v6, v7, v8
+; GFX9-DL-NEXT: v_mad_i32_i24 v3, v3, v4, v2
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v7, v9, v10
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v8, v11, v12
+; GFX9-DL-NEXT: v_add3_u32 v3, v3, v5, v6
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v9, v13, v14
+; GFX9-DL-NEXT: v_mul_i32_i24_e32 v10, v15, v16
+; GFX9-DL-NEXT: v_add3_u32 v3, v3, v7, v8
+; GFX9-DL-NEXT: v_add3_u32 v3, v3, v9, v10
+; GFX9-DL-NEXT: v_add3_u32 v1, v3, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_multiuses_mul1:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_clause 0x1
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v0, v1, 0, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v3, v1, 4, 4
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v4, v2, 4, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v5, v1, 8, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v6, v2, 8, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v7, v2, 0, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v8, v1, 12, 4
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v3, v3, v4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v9, v2, 12, 4
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v4, v5, v6
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40000
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40000
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v0, s4, s5, v0
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v0
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40004
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40004
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40008
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x4000c
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x4000c
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40010
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40010
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40014
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40014
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40018
-; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40018
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s0, 28
-; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s1, 28
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
-; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX10-DL-XNACK-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v5, v0, v7, s2
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v15, v1, 16, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v10, v2, 16, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v11, v1, 20, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v12, v2, 20, 4
+; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v14, v0, v7, v5
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v7, v1, 24, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_i32 v13, v2, 24, 4
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v6, v15, v10
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v8, v8, v9
+; GFX10-DL-XNACK-NEXT: v_add3_u32 v15, v14, v3, v4
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v3, v11, v12
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v4, v7, v13
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i32_e32 v1, 28, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i32_e32 v2, 28, v2
+; GFX10-DL-XNACK-NEXT: v_add3_u32 v6, v15, v8, v6
+; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v7, v1, v2
+; GFX10-DL-XNACK-NEXT: v_add3_u32 v0, v6, v3, v4
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-XNACK-NEXT: v_add3_u32 v0, v0, v7, v5
+; GFX10-DL-XNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_multiuses_mul1:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v2, v1, 0, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v3, v1, 4, 4
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v4, v0, 4, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v5, v1, 8, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v6, v0, 8, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v7, v0, 0, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v8, v1, 12, 4
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v3, v3, v4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v9, v0, 12, 4
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v4, v5, v6
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40000
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40000
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v0, s2, s3, v0
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v0
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40004
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40004
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40008
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x4000c
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x4000c
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28
-; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
-; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v5, v2, v7, s2
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v15, v1, 16, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v10, v0, 16, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v11, v1, 20, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v12, v0, 20, 4
+; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v2, v2, v7, v5
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v7, v1, 24, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v13, v0, 24, 4
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v6, v15, v10
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v8, v8, v9
+; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v15, v2, v3, v4
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v3, v11, v12
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v4, v7, v13
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i32_e32 v1, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i32_e32 v0, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v2, v15, v8, v6
+; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v0, v1, v0
+; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v3, v2, v3, v4
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v0, v3, v0, v5
+; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_multiuses_mul1:
; GFX10-DL: ; %bb.0: ; %entry
@@ -1620,8 +1835,11 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i32
@@ -1691,105 +1909,104 @@ entry:
define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_ashr_i32 s13, s5, 28
-; GFX7-NEXT: s_bfe_i32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_i32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40004
-; GFX7-NEXT: s_bfe_i32 s5, s5, 0x40000
-; GFX7-NEXT: s_ashr_i32 s6, s4, 28
-; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_i32 s12, s4, 0x40004
-; GFX7-NEXT: s_bfe_i32 s4, s4, 0x40000
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s18
-; GFX7-NEXT: v_mad_i32_i24 v0, s11, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s17
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s16
-; GFX7-NEXT: v_mad_i32_i24 v0, s9, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s15
-; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s14
-; GFX7-NEXT: v_mad_i32_i24 v0, s7, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_i32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_i32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_i32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_i32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_i32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_i32 v8, v2, 4, 4
+; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_ashrrev_i32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_i32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 4, 4
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 4
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, s4
+; GFX7-NEXT: v_mad_i32_i24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_i32_i24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot8_acc32_vecMul:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[2:3]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 28, v0
+; GFX8-NEXT: v_bfe_i32 v3, v0, 24, 4
+; GFX8-NEXT: v_bfe_i32 v4, v0, 20, 4
+; GFX8-NEXT: v_bfe_i32 v5, v0, 16, 4
+; GFX8-NEXT: v_bfe_i32 v6, v0, 12, 4
+; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
+; GFX8-NEXT: v_bfe_i32 v8, v0, 4, 4
+; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_ashrrev_i32_e32 v9, 28, v1
+; GFX8-NEXT: v_bfe_i32 v10, v1, 24, 4
+; GFX8-NEXT: v_bfe_i32 v11, v1, 20, 4
+; GFX8-NEXT: v_bfe_i32 v12, v1, 16, 4
+; GFX8-NEXT: v_bfe_i32 v13, v1, 12, 4
+; GFX8-NEXT: v_bfe_i32 v14, v1, 8, 4
+; GFX8-NEXT: v_bfe_i32 v15, v1, 4, 4
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s4, s2, 28
-; GFX8-NEXT: s_ashr_i32 s11, s3, 28
-; GFX8-NEXT: s_bfe_i32 s12, s3, 0x40018
-; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40014
-; GFX8-NEXT: s_bfe_i32 s14, s3, 0x40010
-; GFX8-NEXT: s_bfe_i32 s15, s3, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s16, s3, 0x40008
-; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40004
-; GFX8-NEXT: s_bfe_i32 s3, s3, 0x40000
-; GFX8-NEXT: s_bfe_i32 s5, s2, 0x40018
-; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40014
-; GFX8-NEXT: s_bfe_i32 s7, s2, 0x40010
-; GFX8-NEXT: s_bfe_i32 s8, s2, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s9, s2, 0x40008
-; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40004
-; GFX8-NEXT: s_bfe_i32 s2, s2, 0x40000
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_i32_i24 v0, s2, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: v_mad_i32_i24 v0, s10, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mad_i32_i24 v0, s9, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s15
-; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s14
-; GFX8-NEXT: v_mad_i32_i24 v0, s7, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mad_i32_i24 v0, s5, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: v_mad_i32_i24 v2, s4, v1, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v0, v1, s2
+; GFX8-NEXT: v_mad_i32_i24 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v4, v11, v0
+; GFX8-NEXT: v_mad_i32_i24 v0, v3, v10, v0
+; GFX8-NEXT: v_mad_i32_i24 v2, v2, v9, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1797,120 +2014,117 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: idot8_acc32_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 28, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_ashrrev_i32_e32 v10, 28, v2
+; GFX9-NEXT: v_bfe_i32 v4, v1, 24, 4
+; GFX9-NEXT: v_bfe_i32 v11, v2, 24, 4
+; GFX9-NEXT: v_bfe_i32 v5, v1, 20, 4
+; GFX9-NEXT: v_bfe_i32 v12, v2, 20, 4
+; GFX9-NEXT: v_bfe_i32 v6, v1, 16, 4
+; GFX9-NEXT: v_bfe_i32 v13, v2, 16, 4
+; GFX9-NEXT: v_bfe_i32 v7, v1, 12, 4
+; GFX9-NEXT: v_bfe_i32 v14, v2, 12, 4
+; GFX9-NEXT: v_bfe_i32 v8, v1, 8, 4
+; GFX9-NEXT: v_bfe_i32 v9, v1, 4, 4
+; GFX9-NEXT: v_bfe_i32 v15, v2, 8, 4
+; GFX9-NEXT: v_bfe_i32 v16, v2, 4, 4
+; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 4
+; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 4
+; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
+; GFX9-NEXT: v_mul_i32_i24_e32 v2, v9, v16
+; GFX9-NEXT: v_mul_i32_i24_e32 v8, v8, v15
+; GFX9-NEXT: v_mul_i32_i24_e32 v7, v7, v14
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s4, s0, 28
-; GFX9-NEXT: s_ashr_i32 s11, s1, 28
-; GFX9-NEXT: s_bfe_i32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_i32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_i32 s15, s1, 0x4000c
-; GFX9-NEXT: s_bfe_i32 s16, s1, 0x40008
-; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40004
-; GFX9-NEXT: s_bfe_i32 s1, s1, 0x40000
-; GFX9-NEXT: s_bfe_i32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_i32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_i32 s8, s0, 0x4000c
-; GFX9-NEXT: s_bfe_i32 s9, s0, 0x40008
-; GFX9-NEXT: s_bfe_i32 s10, s0, 0x40004
-; GFX9-NEXT: s_bfe_i32 s0, s0, 0x40000
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_i32_i24 v1, s0, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s17
-; GFX9-NEXT: v_mad_i32_i24 v1, s10, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mad_i32_i24 v1, s9, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s15
-; GFX9-NEXT: v_mad_i32_i24 v1, s8, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_mad_i32_i24 v1, s7, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s13
-; GFX9-NEXT: v_mad_i32_i24 v1, s6, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_mad_i32_i24 v1, s5, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_i32_i24 v1, s4, v2, v1
+; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
+; GFX9-NEXT: v_mul_i32_i24_e32 v6, v6, v13
+; GFX9-NEXT: v_mul_i32_i24_e32 v5, v5, v12
+; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
+; GFX9-NEXT: v_mul_i32_i24_e32 v4, v4, v11
+; GFX9-NEXT: v_mul_i32_i24_e32 v3, v3, v10
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
+; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s14, -1
-; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s12, s12, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot8_i32_i4 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot8_i32_i4 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc32_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_clause 0x1
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
-; GFX10-DL-XNACK-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v1, v1, v2, s2
+; GFX10-DL-XNACK-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc32_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
-; GFX10-DL-NOXNACK-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, v1, v0, s2
+; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
@@ -1936,8 +2150,11 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i32>
%cvec2 = sext <8 x i4> %vec2 to <8 x i32>
@@ -1970,67 +2187,83 @@ entry:
define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_mov_b32 s8, 0xffff
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_ashr_i32 s6, s4, 28
-; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40018
-; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40014
-; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40010
-; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_i32 s19, s5, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40000
-; GFX7-NEXT: s_ashr_i32 s14, s5, 28
-; GFX7-NEXT: s_bfe_i32 s5, s5, 0x40004
-; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40014
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40010
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v4, s18
-; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
-; GFX7-NEXT: v_mov_b32_e32 v3, s19
-; GFX7-NEXT: s_bfe_i32 s13, s4, 0x40000
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
-; GFX7-NEXT: s_bfe_i32 s4, s4, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mul_i32_i24_e32 v1, s4, v1
-; GFX7-NEXT: v_mul_i32_i24_e32 v2, s13, v2
-; GFX7-NEXT: v_mul_i32_i24_e32 v3, s12, v3
-; GFX7-NEXT: v_mul_i32_i24_e32 v4, s11, v4
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s4, 0xffff
+; GFX7-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v1, v3, 20, 4
+; GFX7-NEXT: v_bfe_i32 v4, v3, 16, 4
+; GFX7-NEXT: v_bfe_i32 v5, v3, 4, 4
+; GFX7-NEXT: v_bfe_i32 v6, v3, 0, 4
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v4, s8, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v1
-; GFX7-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v5, s17
-; GFX7-NEXT: v_mov_b32_e32 v6, s16
-; GFX7-NEXT: v_mov_b32_e32 v7, s15
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s10, v5, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s9, v6, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s14
-; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
+; GFX7-NEXT: v_bfe_i32 v10, v0, 20, 4
+; GFX7-NEXT: v_bfe_i32 v11, v0, 16, 4
+; GFX7-NEXT: v_bfe_i32 v12, v0, 4, 4
+; GFX7-NEXT: v_bfe_i32 v13, v0, 0, 4
+; GFX7-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX7-NEXT: v_or_b32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v11
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v12
+; GFX7-NEXT: v_and_b32_e32 v11, v2, v13
+; GFX7-NEXT: v_bfe_i32 v7, v3, 24, 4
+; GFX7-NEXT: v_bfe_i32 v8, v3, 8, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v9, 28, v3
+; GFX7-NEXT: v_bfe_i32 v3, v3, 12, 4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 24, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 8, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v16, 28, v0
+; GFX7-NEXT: v_bfe_i32 v0, v0, 12, 4
+; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v11, v10
+; GFX7-NEXT: v_and_b32_e32 v12, v2, v14
+; GFX7-NEXT: v_and_b32_e32 v13, v2, v15
+; GFX7-NEXT: v_and_b32_e32 v14, v2, v16
+; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v6
+; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
+; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX7-NEXT: v_and_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_and_b32_e32 v1, v2, v1
+; GFX7-NEXT: v_and_b32_e32 v6, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX7-NEXT: v_and_b32_e32 v2, v2, v5
+; GFX7-NEXT: buffer_load_ushort v5, off, s[0:3], 0
+; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v4, v4, v6, v5
+; GFX7-NEXT: v_mad_u32_u24 v4, v16, v11, v4
+; GFX7-NEXT: v_mad_u32_u24 v4, v8, v13, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v2, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v15, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v9, v14, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2038,344 +2271,422 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, 12
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_i32 s8, s0, 0x40000
-; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40000
-; GFX8-NEXT: s_bfe_i32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_i32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_i32 s13, s1, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s14, s1, 0x40004
-; GFX8-NEXT: s_ashr_i32 s9, s1, 28
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: s_ashr_i32 s2, s0, 28
-; GFX8-NEXT: s_bfe_i32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_i32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_i32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_i32 s6, s0, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s7, s0, 0x40004
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: v_mul_i32_i24_e32 v3, s0, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_ushort v4, v[0:1]
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 4, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 20, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 4, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 20, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v15, 28, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v16, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v17, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v5, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 12, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 12, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_i32_i24 v2, s8, v4, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s7, v5, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT: v_mad_i32_i24 v2, s6, v6, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_i32_i24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT: v_mad_u16 v2, v7, v12, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v17
+; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX8-NEXT: v_mad_u16 v2, v8, v13, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
+; GFX8-NEXT: v_mad_u16 v2, v17, v5, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
+; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
+; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v16
+; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v18
+; GFX8-NEXT: v_mad_u16 v2, v9, v14, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
+; GFX8-NEXT: v_mad_u16 v2, v16, v18, v2
+; GFX8-NEXT: v_mad_u16 v2, v10, v15, v2
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: idot8_acc16_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s1, s0, 0x40018
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c
-; GFX9-NEXT: s_and_b32 s11, s0, 15
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s11, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s5, s8
-; GFX9-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s4
-; GFX9-NEXT: s_bfe_u32 s7, s6, 0x40018
-; GFX9-NEXT: s_lshr_b32 s12, s6, 28
-; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40010
-; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40014
-; GFX9-NEXT: s_bfe_u32 s15, s6, 0x40008
-; GFX9-NEXT: s_bfe_u32 s16, s6, 0x4000c
-; GFX9-NEXT: s_and_b32 s17, s6, 15
-; GFX9-NEXT: s_bfe_u32 s6, s6, 0x40004
-; GFX9-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s17, s6
-; GFX9-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v5
-; GFX9-NEXT: global_load_ushort v5, v0, s[2:3]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s15, s16
-; GFX9-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s13, s14
-; GFX9-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v6
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s7, s12
+; GFX9-NEXT: global_load_dword v3, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v4, v0, s[6:7]
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v17, 15, v4
+; GFX9-NEXT: v_bfe_u32 v0, v3, 24, 4
+; GFX9-NEXT: v_bfe_u32 v6, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v13, v4, 16, 4
+; GFX9-NEXT: v_bfe_u32 v15, v4, 8, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v3
+; GFX9-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v9, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v3, v3, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v10, v2, v10
+; GFX9-NEXT: v_bfe_u32 v11, v4, 24, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 28, v4
+; GFX9-NEXT: v_bfe_u32 v14, v4, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v4, 12, 4
+; GFX9-NEXT: v_bfe_u32 v4, v4, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v17, v2, v17
+; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v10
+; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v17
+; GFX9-NEXT: v_pk_lshlrev_b16 v3, 12, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v4
+; GFX9-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-NEXT: v_and_b32_e32 v8, v2, v8
+; GFX9-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
+; GFX9-NEXT: v_lshl_or_b32 v5, v16, 16, v15
+; GFX9-NEXT: v_and_b32_e32 v6, v2, v6
+; GFX9-NEXT: v_pk_lshlrev_b16 v8, 12, v8 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-NEXT: v_and_b32_e32 v13, v2, v13
+; GFX9-NEXT: v_and_b32_e32 v2, v2, v11
+; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v7, v14, 16, v13
+; GFX9-NEXT: v_lshl_or_b32 v2, v12, 16, v2
; GFX9-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_mul_lo_u16 v4, v4, v8
+; GFX9-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_mul_lo_u16 v5, v8, v5
+; GFX9-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX9-NEXT: v_pk_mul_lo_u16 v2, v6, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v5, v1, v5
-; GFX9-NEXT: v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NEXT: v_add_u16_e32 v4, v3, v4
+; GFX9-NEXT: v_add_u16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v3, v3, v5
+; GFX9-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v3, v3, v2
+; GFX9-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v0
+; GFX9-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_nop 0
-; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s1, s0, 0x40018
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x4000c
-; GFX9-DL-NEXT: s_and_b32 s11, s0, 15
-; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x40004
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s11, s0
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v1, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s5, s8
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s1, s4
-; GFX9-DL-NEXT: s_bfe_u32 s7, s6, 0x40018
-; GFX9-DL-NEXT: s_lshr_b32 s12, s6, 28
-; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x4000c
-; GFX9-DL-NEXT: s_and_b32 s17, s6, 15
-; GFX9-DL-NEXT: s_bfe_u32 s6, s6, 0x40004
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s17, s6
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
-; GFX9-DL-NEXT: global_load_ushort v5, v0, s[2:3]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s15, s16
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s13, s14
-; GFX9-DL-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s7, s12
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v4, v0, s[6:7]
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v4
+; GFX9-DL-NEXT: v_bfe_u32 v0, v3, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v4, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v4, 8, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v3
+; GFX9-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v3, v3, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v10, v2, v10
+; GFX9-DL-NEXT: v_bfe_u32 v11, v4, 24, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 28, v4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v4, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v4, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v4, v4, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v17, v2, v17
+; GFX9-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v10
+; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v17
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v3, 12, v3 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
+; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-DL-NEXT: v_and_b32_e32 v8, v2, v8
+; GFX9-DL-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX9-DL-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX9-DL-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-DL-NEXT: v_lshl_or_b32 v8, v9, 16, v8
+; GFX9-DL-NEXT: v_lshl_or_b32 v5, v16, 16, v15
+; GFX9-DL-NEXT: v_and_b32_e32 v6, v2, v6
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v8, 12, v8 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_and_b32_e32 v13, v2, v13
+; GFX9-DL-NEXT: v_and_b32_e32 v2, v2, v11
+; GFX9-DL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-DL-NEXT: v_lshl_or_b32 v7, v14, 16, v13
+; GFX9-DL-NEXT: v_lshl_or_b32 v2, v12, 16, v2
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v8
+; GFX9-DL-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v8, v5
+; GFX9-DL-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v6, v7
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v5, v1, v5
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add_u16_e32 v4, v3, v4
+; GFX9-DL-NEXT: v_add_u16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v5
+; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v2
+; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v0
+; GFX9-DL-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc16_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v4, 0xffff
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s6, 28
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s2, s6, 0x40010
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s3, s6, 0x40014
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s8, s6, 0x40008
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s9, s6, 0x4000c
-; GFX10-DL-XNACK-NEXT: s_and_b32 s10, s6, 15
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s6, 0x40004
-; GFX10-DL-XNACK-NEXT: s_and_b32 s11, s7, 15
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s10, s6
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s10, s7, 0x40004
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, s6 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s11, s10
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s11, s7, 0x4000c
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v3, 12, s6 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s7, 0x40008
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-XNACK-NEXT: global_load_ushort v3, v0, s[0:1]
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v11, 15, v1
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, 15, v2
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v7, v1, 16, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v9, v1, 8, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v5, v1, 24, 4
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v10, v1, 12, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v1, v1, 4, 4
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v11, v4, v11
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v16, v2, 4, 4
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, v4, v13
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v18, v2, 8, 4
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v9, v4, v9
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v11
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v7, v4, v7
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v11, v16, 16, v13
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v19, v2, 24, 4
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 28, v2
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v15, v2, 16, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v17, v2, 20, 4
+; GFX10-DL-XNACK-NEXT: v_bfe_u32 v2, v2, 12, 4
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, v4, v18
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v11, 12, v11 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v9, v10, 16, v9
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v7, v8, 16, v7
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v2, v2, 16, v13
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v8, 12, v11 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v10, v4, v15
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v8
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v8, v17, 16, v10
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, v4, v5
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, v8 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v4, v4, v19
+; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v9, v2
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v10
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v3, v6, 16, v3
+; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v4, v14, 16, v4
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v2
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, v3 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v3, 12, v4 op_sel_hi:[0,1]
+; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v4, v7, v5
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v6
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s8, s8, s9
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s6, s11
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v4, 12, s8 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s6 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s8, s7, 0x40010
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s7, 0x40014
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v7, v1, v4
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v3
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v4 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s3, s8, s6
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s3 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-XNACK-NEXT: s_bfe_u32 s10, s7, 0x40018
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s2, s7, 28
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s1, s10, s2
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s1 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v4
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v7, v5
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v2
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc16_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v4, 0xffff
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s2, s0, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s0, 28
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s6, s0, 0x40010
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s7, s0, 0x40014
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s9, s0, 0x4000c
-; GFX10-DL-NOXNACK-NEXT: s_and_b32 s10, s0, 15
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s0, 0x40004
-; GFX10-DL-NOXNACK-NEXT: s_and_b32 s11, s1, 15
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s10, s0
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s10, s1, 0x40004
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s11, s10
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s11, s1, 0x4000c
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s1, 0x40008
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s8, s8, s9
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s0, s11
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v4, 12, s8 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s8, s1, 0x40010
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s1, 0x40014
-; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v3
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v4 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s6, s6, s7
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s8, s0
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s6 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s0, s1, 28
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s1, s2, s3
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s10, s0
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: global_load_ushort v3, v2, s[0:1]
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v11, 15, v1
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, 15, v0
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v7, v1, 16, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v9, v1, 8, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v5, v1, 24, 4
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v10, v1, 12, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v1, v1, 4, 4
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v11, v4, v11
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v16, v0, 4, 4
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, v4, v13
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v18, v0, 8, 4
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v9, v4, v9
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v11
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v7, v4, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v11, v16, 16, v13
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v19, v0, 24, 4
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v15, v0, 16, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v17, v0, 20, 4
+; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v0, v0, 12, 4
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, v4, v18
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v11, 12, v11 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v9, v10, 16, v9
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v7, v8, 16, v7
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v0, v0, 16, v13
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v8, 12, v11 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v10, v4, v15
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v8
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v8, v17, 16, v10
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 16, v1
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s1 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v4
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v5 op_sel_hi:[0,1]
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v3
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NOXNACK-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, v4, v5
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, v8 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v4, v4, v19
+; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v0, v9, v0
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v1, v1, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v3, v6, 16, v3
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v4, v14, 16, v4
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v1, v0
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v3 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v3, 12, v4 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v4, v7, v5
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v7, v0, v6
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v7, v4
+; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v3
+; GFX10-DL-NOXNACK-NEXT: global_store_short v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
@@ -2451,8 +2762,11 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i16>
%cvec2 = sext <8 x i4> %vec2 to <8 x i16>
@@ -2485,88 +2799,109 @@ entry:
define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_movk_i32 s8, 0xff
-; GFX7-NEXT: s_mov_b32 s9, 0xffff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
-; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40000
-; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40004
-; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40008
-; GFX7-NEXT: s_bfe_i32 s18, s5, 0x4000c
-; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40010
-; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40014
-; GFX7-NEXT: s_bfe_i32 s21, s5, 0x40018
-; GFX7-NEXT: s_ashr_i32 s5, s5, 28
-; GFX7-NEXT: v_mov_b32_e32 v8, s15
-; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v7, s16
-; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v6, s17
-; GFX7-NEXT: s_bfe_i32 s11, s4, 0x4000c
-; GFX7-NEXT: v_mov_b32_e32 v5, s18
-; GFX7-NEXT: s_bfe_i32 s12, s4, 0x40010
-; GFX7-NEXT: v_mov_b32_e32 v4, s19
-; GFX7-NEXT: s_bfe_i32 s13, s4, 0x40014
-; GFX7-NEXT: v_mov_b32_e32 v3, s20
-; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40018
-; GFX7-NEXT: v_mov_b32_e32 v2, s21
-; GFX7-NEXT: s_ashr_i32 s4, s4, 28
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mul_i32_i24_e32 v1, s4, v1
-; GFX7-NEXT: v_mul_i32_i24_e32 v2, s14, v2
-; GFX7-NEXT: v_mul_i32_i24_e32 v3, s13, v3
-; GFX7-NEXT: v_mul_i32_i24_e32 v9, s12, v4
-; GFX7-NEXT: v_mul_i32_i24_e32 v5, s11, v5
-; GFX7-NEXT: v_mul_i32_i24_e32 v6, s10, v6
-; GFX7-NEXT: v_mul_i32_i24_e32 v7, s7, v7
-; GFX7-NEXT: v_mul_i32_i24_e32 v8, s6, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_and_b32_e32 v9, s8, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_and_b32_e32 v6, s8, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; GFX7-NEXT: v_and_b32_e32 v8, s8, v8
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v9, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v6, v5
-; GFX7-NEXT: v_or_b32_e32 v5, v8, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v2, s9, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v5, s9, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX7-NEXT: v_alignbit_b32 v3, v1, v2, 8
-; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_movk_i32 s4, 0xff
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX7-NEXT: s_mov_b32 s5, 0xffff
+; GFX7-NEXT: v_mov_b32_e32 v3, 0xffff
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_bfe_i32 v6, v4, 20, 4
+; GFX7-NEXT: v_bfe_i32 v7, v4, 16, 4
+; GFX7-NEXT: v_bfe_i32 v8, v4, 12, 4
+; GFX7-NEXT: v_bfe_i32 v9, v4, 8, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX7-NEXT: v_and_b32_e32 v9, s4, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT: v_mad_i32_i24 v0, s12, v4, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_bfe_i32 v13, v0, 24, 4
+; GFX7-NEXT: v_bfe_i32 v16, v0, 12, 4
+; GFX7-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v9, v8
+; GFX7-NEXT: v_and_b32_e32 v9, v2, v13
+; GFX7-NEXT: v_lshlrev_b32_e32 v13, 8, v16
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: v_bfe_i32 v5, v4, 24, 4
+; GFX7-NEXT: v_bfe_i32 v10, v4, 4, 4
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 28, v4
+; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v11, 8, v1
+; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT: v_ashrrev_i32_e32 v12, 28, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 8, v10
+; GFX7-NEXT: v_and_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
+; GFX7-NEXT: v_bfe_i32 v15, v0, 16, 4
+; GFX7-NEXT: v_bfe_i32 v17, v0, 8, 4
+; GFX7-NEXT: v_bfe_i32 v18, v0, 4, 4
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 4
+; GFX7-NEXT: v_or_b32_e32 v5, v5, v11
+; GFX7-NEXT: v_or_b32_e32 v4, v4, v10
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v12
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX7-NEXT: v_and_b32_e32 v11, v2, v15
+; GFX7-NEXT: v_and_b32_e32 v14, v2, v17
+; GFX7-NEXT: v_lshlrev_b32_e32 v15, 8, v18
+; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v15
+; GFX7-NEXT: v_or_b32_e32 v8, v9, v8
+; GFX7-NEXT: v_or_b32_e32 v9, v11, v10
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_and_b32_e32 v6, s5, v6
+; GFX7-NEXT: v_or_b32_e32 v10, v14, v13
+; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_and_b32_e32 v0, v3, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_and_b32_e32 v4, s5, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-NEXT: v_or_b32_e32 v4, v4, v7
+; GFX7-NEXT: v_and_b32_e32 v7, v3, v9
+; GFX7-NEXT: v_or_b32_e32 v3, v7, v6
+; GFX7-NEXT: v_and_b32_e32 v7, v2, v4
+; GFX7-NEXT: v_and_b32_e32 v13, v2, v0
+; GFX7-NEXT: v_bfe_u32 v8, v4, 8, 8
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 8
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v0
+; GFX7-NEXT: v_bfe_u32 v4, v4, 16, 8
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX7-NEXT: v_and_b32_e32 v1, v2, v1
+; GFX7-NEXT: v_and_b32_e32 v12, v2, v12
+; GFX7-NEXT: v_and_b32_e32 v9, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v2, v2, v3
+; GFX7-NEXT: v_bfe_u32 v10, v5, 8, 8
+; GFX7-NEXT: v_bfe_u32 v15, v3, 8, 8
+; GFX7-NEXT: v_bfe_u32 v5, v5, 16, 8
+; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 8
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v7, v7, v13, v16
+; GFX7-NEXT: v_mad_u32_u24 v7, v8, v14, v7
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v0, v7
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v9, v2, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v10, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v3, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v12, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2574,456 +2909,488 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, 12
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
-; GFX8-NEXT: s_mov_b32 s0, 0xffff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004
-; GFX8-NEXT: s_bfe_i32 s9, s1, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40004
-; GFX8-NEXT: s_bfe_i32 s15, s2, 0x40000
-; GFX8-NEXT: s_bfe_i32 s16, s2, 0x4000c
-; GFX8-NEXT: s_bfe_i32 s3, s1, 0x40014
-; GFX8-NEXT: s_ashr_i32 s5, s1, 28
-; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40014
-; GFX8-NEXT: s_bfe_i32 s11, s2, 0x40010
-; GFX8-NEXT: s_ashr_i32 s12, s2, 28
-; GFX8-NEXT: s_bfe_i32 s13, s2, 0x40018
-; GFX8-NEXT: s_bfe_i32 s2, s2, 0x40008
-; GFX8-NEXT: s_bfe_i32 s8, s1, 0x40000
-; GFX8-NEXT: v_mov_b32_e32 v4, s16
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
-; GFX8-NEXT: v_mov_b32_e32 v6, s15
-; GFX8-NEXT: v_mov_b32_e32 v7, s14
-; GFX8-NEXT: v_mov_b32_e32 v8, s7
-; GFX8-NEXT: v_mul_i32_i24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mul_i32_i24_e32 v5, s8, v6
-; GFX8-NEXT: v_mul_i32_i24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_bfe_i32 s4, s1, 0x40010
-; GFX8-NEXT: s_bfe_i32 s6, s1, 0x40018
-; GFX8-NEXT: v_mov_b32_e32 v9, s13
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mov_b32_e32 v10, s12
-; GFX8-NEXT: v_mov_b32_e32 v11, s5
-; GFX8-NEXT: v_mov_b32_e32 v12, s11
-; GFX8-NEXT: v_mov_b32_e32 v13, s10
-; GFX8-NEXT: v_mov_b32_e32 v14, s3
-; GFX8-NEXT: v_mul_i32_i24_e32 v3, s1, v3
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mul_i32_i24_e32 v7, s6, v9
-; GFX8-NEXT: v_mul_i32_i24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mul_i32_i24_e32 v9, s4, v12
-; GFX8-NEXT: v_mul_i32_i24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_e32 v5, s0, v5
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_e32 v4, s0, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v6, v4, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v6
+; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 28, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 20, v3
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v17, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v18, 12, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v5, 12, v10
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v16
+; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
+; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 12, v6
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v15
+; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v18
+; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
+; GFX8-NEXT: v_ashrrev_i16_e32 v19, 12, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 12, v11
+; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
+; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v14
+; GFX8-NEXT: v_mul_lo_u16_e32 v10, v10, v15
+; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT: v_mul_lo_u16_sdwa v3, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_mul_lo_u16_e32 v15, v16, v18
+; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v8, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_mul_lo_u16_e32 v8, v9, v11
+; GFX8-NEXT: v_or_b32_sdwa v3, v15, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT: v_mul_lo_u16_e32 v14, v17, v19
+; GFX8-NEXT: v_mul_lo_u16_sdwa v5, v5, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v6, v14, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_or_b32_sdwa v8, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v3
+; GFX8-NEXT: v_or_b32_sdwa v3, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v3
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 24, v[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v5
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v8, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v3, v8, v4
+; GFX8-NEXT: v_add_u16_e32 v3, v3, v5
+; GFX8-NEXT: v_add_u16_e32 v3, v3, v7
+; GFX8-NEXT: v_add_u16_e32 v2, v3, v2
+; GFX8-NEXT: v_mad_u16 v2, v17, v19, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v6
+; GFX8-NEXT: v_mad_u16 v2, v16, v18, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v10
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: idot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_mov_b32 s0, 0xffff
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s9, s1, 4
-; GFX9-NEXT: s_lshr_b32 s16, s8, 4
-; GFX9-NEXT: v_lshlrev_b16_e64 v2, 12, s1
-; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s8
-; GFX9-NEXT: v_lshlrev_b16_e64 v6, 12, s9
-; GFX9-NEXT: v_lshlrev_b16_e64 v13, 12, s16
-; GFX9-NEXT: s_lshr_b32 s10, s1, 12
-; GFX9-NEXT: s_lshr_b32 s11, s1, 8
-; GFX9-NEXT: s_lshr_b32 s17, s8, 12
-; GFX9-NEXT: s_lshr_b32 s18, s8, 8
-; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s11
-; GFX9-NEXT: v_lshlrev_b16_e64 v5, 12, s10
-; GFX9-NEXT: v_lshlrev_b16_e64 v11, 12, s18
-; GFX9-NEXT: v_lshlrev_b16_e64 v12, 12, s17
-; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
-; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v4, v3, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 12
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 20, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 20, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_lshlrev_b16_e32 v17, 12, v2
+; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 12, v9
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 12, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v14
+; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-NEXT: v_ashrrev_i16_e32 v18, 12, v0
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 12, v10
+; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
-; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-NEXT: v_ashrrev_i16_e32 v0, 12, v0
; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-NEXT: v_mul_lo_u16_e32 v2, v2, v3
-; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_lshr_b32 s4, s1, 20
-; GFX9-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-NEXT: s_lshr_b32 s12, s8, 20
-; GFX9-NEXT: s_lshr_b32 s13, s8, 16
-; GFX9-NEXT: v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v4, v4, v11
-; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s5
-; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-NEXT: v_lshlrev_b16_e64 v16, 12, s13
-; GFX9-NEXT: v_lshlrev_b16_e64 v17, 12, s12
-; GFX9-NEXT: s_lshr_b32 s6, s1, 28
-; GFX9-NEXT: s_lshr_b32 s7, s1, 24
-; GFX9-NEXT: s_lshr_b32 s14, s8, 28
-; GFX9-NEXT: s_lshr_b32 s15, s8, 24
-; GFX9-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e64 v7, 12, s7
-; GFX9-NEXT: v_lshlrev_b16_e64 v8, 12, s6
-; GFX9-NEXT: v_lshlrev_b16_e64 v14, 12, s15
-; GFX9-NEXT: v_lshlrev_b16_e64 v15, 12, s14
-; GFX9-NEXT: v_or_b32_e32 v4, v2, v4
-; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v17
-; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v13
+; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-NEXT: v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v16
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 8, v4
-; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v7, v7, v14
-; GFX9-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX9-NEXT: v_or_b32_e32 v5, v3, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v7, v8, v10
+; GFX9-NEXT: v_or_b32_sdwa v1, v19, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v2, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v5, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v6
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_add_u16_e32 v1, v7, v4
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v6
+; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v16, v18, v0
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v5
+; GFX9-NEXT: v_mad_legacy_u16 v0, v15, v17, v0
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v9
+; GFX9-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s9, s1, 4
-; GFX9-DL-NEXT: s_lshr_b32 s16, s8, 4
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v2, 12, s1
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s8
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v6, 12, s9
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v13, 12, s16
-; GFX9-DL-NEXT: s_lshr_b32 s10, s1, 12
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 8
-; GFX9-DL-NEXT: s_lshr_b32 s17, s8, 12
-; GFX9-DL-NEXT: s_lshr_b32 s18, s8, 8
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s11
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v5, 12, s10
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v11, 12, s18
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v12, 12, s17
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[2:3]
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 12
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 20, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 20, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v17, 12, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v2, 12, v9
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v1
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v1, 12, v5
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v14
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v18, 12, v0
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v0, 12, v10
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v0, 12, v0
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v2, v2, v3
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: s_lshr_b32 s4, s1, 20
-; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16
-; GFX9-DL-NEXT: s_lshr_b32 s12, s8, 20
-; GFX9-DL-NEXT: s_lshr_b32 s13, s8, 16
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, v4, v11
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s5
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v16, 12, s13
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v17, 12, s12
-; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 28
-; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 24
-; GFX9-DL-NEXT: s_lshr_b32 s14, s8, 28
-; GFX9-DL-NEXT: s_lshr_b32 s15, s8, 24
-; GFX9-DL-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX9-DL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v7, 12, s7
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v8, 12, s6
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v14, 12, s15
-; GFX9-DL-NEXT: v_lshlrev_b16_e64 v15, 12, s14
-; GFX9-DL-NEXT: v_or_b32_e32 v4, v2, v4
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v17
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v13
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v16
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v4
-; GFX9-DL-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v14
-; GFX9-DL-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v3
-; GFX9-DL-NEXT: v_or_b32_e32 v5, v3, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v8, v10
+; GFX9-DL-NEXT: v_or_b32_sdwa v1, v19, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v2, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
+; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX9-DL-NEXT: v_or_b32_sdwa v1, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v6
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v7, v4
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v16, v18, v0
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v5
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v15, v17, v0
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v9
+; GFX9-DL-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-XNACK-LABEL: idot8_acc8_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry
-; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s22, -1
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s23, 0x31c16000
-; GFX10-DL-XNACK-NEXT: s_add_u32 s20, s20, s3
-; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-XNACK-NEXT: s_addc_u32 s21, s21, 0
-; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-XNACK-NEXT: s_mov_b32 s0, 0xffff
+; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v19, 0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s9, s6, 4
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s16, s7, 4
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s9
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s16
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s10, s6, 12
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s17, s7, 12
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s6
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s7
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v13, 12, s17
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, s10
+; GFX10-DL-XNACK-NEXT: s_clause 0x1
+; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v19, s[0:1]
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v15, 12, v15
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 28, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v15
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v0, 20, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 20, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 24, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v8, v8, v15
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v17
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v14, 12, v14
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v11, 12, v11
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v12, 12, v12
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s11, s6, 8
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s18, s7, 8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, s11
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v11, 12, s18
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v19, 12, v5
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v12
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v17
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v9, v9, v16
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 8, v8
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, v12
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v11
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v2, v2, v3
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v6
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v3, v19, v13
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s6, 20
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s2, s6, 16
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s3, s6, 28
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s8, s6, 24
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s12, s7, 20
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, s8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, s3
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, s2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, s1
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s12
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v11
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 8, v3
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s13, s7, 16
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s14, s7, 28
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s13
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v7
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v8
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v9
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v15, 12, s14
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v10
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v12
-; GFX10-DL-XNACK-NEXT: s_lshr_b32 s15, s7, 24
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v15
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v14, 12, s15
-; GFX10-DL-XNACK-NEXT: v_or_b32_e32 v3, v2, v3
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v9
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v15, v8, v6
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v7, v7, v10
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v14
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 8, v3
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v10, v10, v15
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v9, v0, v11
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v23, 12, v12
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v11, v7, v14
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 8, v6
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v1, v1, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 8, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v2, v5, v23
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 8, v9
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v6, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v11, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 8, v4
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v5, v11
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v7
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v8
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v15, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-XNACK-NEXT: v_or_b32_e32 v3, v2, v4
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v3, v1, v3
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 8, v11
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v9, v3, v10
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v9, v8
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v2
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v0, v5, v23, v0
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v6
+; GFX10-DL-XNACK-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-XNACK-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm
;
; GFX10-DL-NOXNACK-LABEL: idot8_acc8_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s22, -1
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s23, 0x31c16000
-; GFX10-DL-NOXNACK-NEXT: s_add_u32 s20, s20, s3
-; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s21, s21, 0
-; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
-; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s2, 0xffff
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v19, 0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s9, s0, 4
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s16, s1, 4
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s9
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s16
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s10, s0, 12
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s17, s1, 12
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v13, 12, s17
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, s10
+; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v2, v19, s[0:1]
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
+; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 20, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 24, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v15, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v16
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 20, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v13, 12, v13
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v8, v8, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, v3
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v11, 12, v11
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v12, 12, v12
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s11, s0, 8
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s18, s1, 8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, s11
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v11, 12, s18
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v19, 12, v5
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v12
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v11
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v2, v2, v3
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v6
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v3, v19, v13
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s0, 20
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s6, s0, 16
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s7, s0, 28
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s8, s0, 24
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s12, s1, 20
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, s8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, s7
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, s6
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, s3
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s12
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v11
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v23, v9, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 8, v8
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v13
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, v12
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v11
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v1
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v10, v10, v15
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v3, v3, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v12
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v9, v7, v14
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 8, v6
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 8, v10
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v1, v1, v18
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v12, v5, v11
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 8, v3
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s13, s1, 16
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s14, s1, 28
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s13
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v7
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v8
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v9
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v15, 12, s14
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v10
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v12
-; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s15, s1, 24
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v15
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v14, 12, s15
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_e32 v3, v2, v3
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v9
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v15, v8, v6
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v7, v7, v10
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v14
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 8, v3
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v6, v9, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v9, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 8, v4
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v5, v11
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v7
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v8
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v2, v15, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_e32 v3, v2, v4
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v3
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NOXNACK-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v2, v1, v2
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v9
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v3, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v9, v2, v9
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v9, v8
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v2
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v5, v11, v0
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v6
+; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-NOXNACK-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
@@ -3124,8 +3491,11 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
<8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i8>
%cvec2 = sext <8 x i4> %vec2 to <8 x i8>
@@ -3153,3 +3523,5 @@ entry:
store i8 %add8, i8 addrspace(1)* %dst, align 4
ret void
}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 157cfec80bc2..d088a195a400 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -9,54 +9,53 @@
define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc32:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s7, s6, 28
-; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
-; GFX7-NEXT: s_and_b32 s6, s6, 15
-; GFX7-NEXT: s_lshr_b32 s5, s4, 28
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s18
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s17
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s16
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s15
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s14
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -64,50 +63,50 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[2:3]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
+; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
+; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
+; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
+; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
+; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
+; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
+; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
+; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s7, s6, 28
-; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX8-NEXT: s_and_b32 s6, s6, 15
-; GFX8-NEXT: s_lshr_b32 s3, s2, 28
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014
-; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010
-; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008
-; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004
-; GFX8-NEXT: s_and_b32 s2, s2, 15
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s15
-; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s14
-; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, s2
+; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v9, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -115,105 +114,104 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: udot8_acc32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 4
+; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 4
+; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 4
+; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v16
+; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s7, s6, 28
-; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX9-NEXT: s_and_b32 s6, s6, 15
-; GFX9-NEXT: s_lshr_b32 s1, s0, 28
-; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s17
-; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s15
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s13
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1
+; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
+; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v10
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
+; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc32:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s14, -1
-; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s12, s12, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc32:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i32
@@ -282,54 +280,53 @@ entry:
define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc16:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v16, off, s[0:3], 0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -337,211 +334,202 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
-; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
-; GFX8-NEXT: s_and_b32 s0, s0, 15
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ushort v18, v[2:3]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 24, 4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX8-NEXT: v_bfe_u32 v12, v0, 24, 4
+; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
+; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40008
-; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40004
-; GFX9-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40004
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_load_ushort v17, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-NEXT: v_bfe_u32 v4, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-NEXT: v_bfe_u32 v11, v3, 24, 4
+; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc16:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s17, s1, 0x40004
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-DL-NEXT: s_and_b32 s1, s1, 15
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40004
-; GFX9-DL-NEXT: s_and_b32 s0, s0, 15
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ushort v17, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc16:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40008
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i16
@@ -610,54 +598,53 @@ entry:
define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -665,211 +652,202 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
-; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
-; GFX8-NEXT: s_and_b32 s0, s0, 15
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 24, 4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX8-NEXT: v_bfe_u32 v12, v0, 24, 4
+; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
+; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc8:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40008
-; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40004
-; GFX9-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40004
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-NEXT: v_bfe_u32 v4, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-NEXT: v_bfe_u32 v11, v3, 24, 4
+; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc8:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s17, s1, 0x40004
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-DL-NEXT: s_and_b32 s1, s1, 15
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40004
-; GFX9-DL-NEXT: s_and_b32 s0, s0, 15
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc8:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40008
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i8
@@ -938,54 +916,53 @@ entry:
define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc4:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
@@ -994,223 +971,206 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s8, s0, 15
-; GFX8-NEXT: s_and_b32 s15, s1, 15
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3
-; GFX8-NEXT: v_and_b32_e32 v3, 15, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v0
+; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
+; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc4:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s10, s0, 15
-; GFX9-NEXT: s_and_b32 s17, s1, 15
-; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
+; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v3
+; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc4:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s10, s0, 15
-; GFX9-DL-NEXT: s_and_b32 s17, s1, 15
-; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v3
+; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc4:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c
-; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s1, s3
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s2, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%v2e0 = extractelement <8 x i4> %vec2, i64 0
@@ -1263,54 +1223,53 @@ entry:
define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_CommutationInsideMAD:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
@@ -1319,223 +1278,206 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s8, s0, 15
-; GFX8-NEXT: s_and_b32 s15, s1, 15
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3
-; GFX8-NEXT: v_and_b32_e32 v3, 15, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v0
+; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
+; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
+; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_CommutationInsideMAD:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s10, s0, 15
-; GFX9-NEXT: s_and_b32 s17, s1, 15
-; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
+; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v3
+; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v2, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_CommutationInsideMAD:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s10, s0, 15
-; GFX9-DL-NEXT: s_and_b32 s17, s1, 15
-; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v3
+; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
+; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_CommutationInsideMAD:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
-; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40008
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c
-; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s0, s1
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s2, s3, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%v2e0 = extractelement <8 x i4> %vec2, i64 0
@@ -1586,56 +1528,55 @@ entry:
define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_multiuses_mul1:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
-; GFX7-NEXT: s_lshr_b32 s7, s6, 28
-; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
-; GFX7-NEXT: s_and_b32 s6, s6, 15
-; GFX7-NEXT: s_lshr_b32 s5, s4, 28
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_u32_u24 v1, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
-; GFX7-NEXT: v_mad_u32_u24 v1, s13, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s18
-; GFX7-NEXT: v_mad_u32_u24 v1, s12, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s17
-; GFX7-NEXT: v_mad_u32_u24 v1, s11, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s16
-; GFX7-NEXT: v_mad_u32_u24 v1, s10, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s15
-; GFX7-NEXT: v_mad_u32_u24 v1, s9, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s14
-; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, s7
-; GFX7-NEXT: v_mad_u32_u24 v1, s5, v2, v1
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v16, v2, v0, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v2, v8, v15, v16
+; GFX7-NEXT: v_mad_u32_u24 v2, v7, v14, v2
+; GFX7-NEXT: v_mad_u32_u24 v2, v6, v13, v2
+; GFX7-NEXT: v_mad_u32_u24 v2, v5, v12, v2
+; GFX7-NEXT: v_mad_u32_u24 v2, v4, v11, v2
+; GFX7-NEXT: v_mad_u32_u24 v2, v3, v10, v2
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v9, v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1643,52 +1584,52 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[2:3]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
+; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
+; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
+; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
+; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
+; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
+; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
+; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
+; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX8-NEXT: s_lshr_b32 s7, s6, 28
-; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX8-NEXT: s_and_b32 s6, s6, 15
-; GFX8-NEXT: s_lshr_b32 s3, s2, 28
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014
-; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010
-; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008
-; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004
-; GFX8-NEXT: s_and_b32 s2, s2, 15
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_u32_u24 v1, s2, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s17
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1
-; GFX8-NEXT: v_mad_u32_u24 v1, s11, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s16
-; GFX8-NEXT: v_mad_u32_u24 v1, s10, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s15
-; GFX8-NEXT: v_mad_u32_u24 v1, s9, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s14
-; GFX8-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s13
-; GFX8-NEXT: v_mad_u32_u24 v1, s5, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s12
-; GFX8-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: v_mad_u32_u24 v1, s3, v2, v1
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v16, v0, v1, s2
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v16
+; GFX8-NEXT: v_mad_u32_u24 v1, v8, v15, v16
+; GFX8-NEXT: v_mad_u32_u24 v1, v7, v14, v1
+; GFX8-NEXT: v_mad_u32_u24 v1, v6, v13, v1
+; GFX8-NEXT: v_mad_u32_u24 v1, v5, v12, v1
+; GFX8-NEXT: v_mad_u32_u24 v1, v4, v11, v1
+; GFX8-NEXT: v_mad_u32_u24 v1, v3, v10, v1
+; GFX8-NEXT: v_mad_u32_u24 v1, v2, v9, v1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -1696,165 +1637,165 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: udot8_multiuses_mul1:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_bfe_u32 v3, v1, 4, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX9-NEXT: v_bfe_u32 v5, v1, 24, 4
+; GFX9-NEXT: v_bfe_u32 v12, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v10, v2, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v17, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX9-NEXT: s_lshr_b32 s7, s6, 28
-; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX9-NEXT: s_and_b32 s6, s6, 15
-; GFX9-NEXT: s_lshr_b32 s1, s0, 28
-; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_u32_u24 v2, s0, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2
-; GFX9-NEXT: v_mad_u32_u24 v2, s11, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_mad_u32_u24 v2, s10, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-NEXT: v_mad_u32_u24 v2, s9, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s14
-; GFX9-NEXT: v_mad_u32_u24 v2, s8, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-NEXT: v_mad_u32_u24 v2, s5, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
-; GFX9-NEXT: v_mad_u32_u24 v2, s4, v3, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT: v_mad_u32_u24 v1, v1, v2, s0
+; GFX9-NEXT: v_mul_u32_u24_e32 v9, v9, v16
+; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-NEXT: v_mad_u32_u24 v2, v3, v10, v1
+; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
+; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-NEXT: v_add3_u32 v2, v2, v9, v8
+; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-NEXT: v_add3_u32 v2, v2, v7, v6
+; GFX9-NEXT: v_add3_u32 v2, v2, v5, v4
+; GFX9-NEXT: v_add3_u32 v1, v17, v1, v2
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_multiuses_mul1:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-DL-NEXT: s_nop 0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_u32 v3, v1, 4, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v17, v1, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX9-DL-NEXT: s_lshr_b32 s7, s6, 28
-; GFX9-DL-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX9-DL-NEXT: s_and_b32 s6, s6, 15
-; GFX9-DL-NEXT: s_lshr_b32 s1, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s4, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s11, s0, 0x40004
-; GFX9-DL-NEXT: s_and_b32 s0, s0, 15
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v1, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v1, v2
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s10, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s12
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s4, v3, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, s0
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v9, v9, v16
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v10, v1
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v14
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-DL-NEXT: v_add3_u32 v2, v2, v9, v8
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-DL-NEXT: v_add3_u32 v2, v2, v7, v6
+; GFX9-DL-NEXT: v_add3_u32 v2, v2, v5, v4
+; GFX9-DL-NEXT: v_add3_u32 v1, v17, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_multiuses_mul1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v8, 15, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_and_b32_e32 v9, 15, v2
+; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v1
+; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: s_and_b32 s4, s0, 15
-; GFX10-DL-NEXT: s_and_b32 s5, s1, 15
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v0
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40008
-; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x4000c
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40010
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1
-; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40018
-; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: v_mad_u32_u24 v13, v8, v9, s2
+; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 16, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v11
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v12
+; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v10, v13
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX10-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v15
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v14
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v7
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v4, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v10
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v3, v8, v9
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v6, v5
+; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_add3_u32 v0, v3, v13, v0
+; GFX10-DL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i32
@@ -1923,54 +1864,53 @@ entry:
define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s26, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
-; GFX7-NEXT: s_add_u32 s24, s24, s3
-; GFX7-NEXT: s_addc_u32 s25, s25, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s7, s6, 28
-; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
-; GFX7-NEXT: s_and_b32 s6, s6, 15
-; GFX7-NEXT: s_lshr_b32 s5, s4, 28
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s18
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s17
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s16
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s15
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s14
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -1978,50 +1918,50 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: flat_load_dword v1, v[2:3]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
+; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
+; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
+; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
+; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
+; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
+; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
+; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
+; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s7, s6, 28
-; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX8-NEXT: s_and_b32 s6, s6, 15
-; GFX8-NEXT: s_lshr_b32 s3, s2, 28
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014
-; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010
-; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008
-; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004
-; GFX8-NEXT: s_and_b32 s2, s2, 15
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mad_u32_u24 v0, s10, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s15
-; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s14
-; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, s2
+; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX8-NEXT: v_mad_u32_u24 v2, v2, v9, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -2029,105 +1969,104 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
;
; GFX9-LABEL: udot8_acc32_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
+; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 4
+; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 4
+; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 4
+; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v16
+; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s7, s6, 28
-; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008
-; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004
-; GFX9-NEXT: s_and_b32 s6, s6, 15
-; GFX9-NEXT: s_lshr_b32 s1, s0, 28
-; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s17
-; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s15
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s13
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1
+; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
+; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v10
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
+; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s14, -1
-; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s12, s12, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s0, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v2, v3, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s0, s1, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i32>
%cvec2 = zext <8 x i4> %vec2 to <8 x i32>
@@ -2161,64 +2100,66 @@ entry:
define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mul_u32_u24_e32 v2, s12, v2
-; GFX7-NEXT: v_mul_u32_u24_e32 v4, s10, v4
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mul_u32_u24_e32 v1, s4, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_u32_u24_e32 v3, s11, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v16, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, 0xf0000
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_bfe_u32 v7, v2, 20, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 12, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 4
+; GFX7-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX7-NEXT: v_alignbit_b32 v2, v7, v2, 16
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 12, v0
+; GFX7-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX7-NEXT: v_and_b32_e32 v7, s4, v8
+; GFX7-NEXT: v_and_b32_e32 v13, 15, v0
+; GFX7-NEXT: v_or_b32_e32 v7, v13, v7
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v7
+; GFX7-NEXT: v_and_b32_e32 v6, 15, v6
+; GFX7-NEXT: v_and_b32_e32 v7, 15, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v6, v6, v7, v16
+; GFX7-NEXT: v_bfe_u32 v12, v0, 8, 4
+; GFX7-NEXT: v_mad_u32_u24 v6, v8, v13, v6
+; GFX7-NEXT: v_bfe_u32 v14, v0, 20, 4
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 12, 4
+; GFX7-NEXT: v_alignbit_b32 v0, v14, v0, 16
+; GFX7-NEXT: v_mad_u32_u24 v5, v5, v12, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX7-NEXT: v_mad_u32_u24 v4, v4, v11, v5
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v4
+; GFX7-NEXT: v_mad_u32_u24 v0, v15, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2226,236 +2167,268 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ushort v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
-; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
-; GFX8-NEXT: s_and_b32 s0, s0, 15
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_ushort v18, v[2:3]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 4, 4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 12, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 16, 4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v11, 15, v0
+; GFX8-NEXT: v_bfe_u32 v12, v0, 4, 4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_mad_u16 v1, v1, v11, v18
+; GFX8-NEXT: v_bfe_u32 v13, v0, 8, 4
+; GFX8-NEXT: v_mad_u16 v1, v5, v12, v1
+; GFX8-NEXT: v_bfe_u32 v14, v0, 12, 4
+; GFX8-NEXT: v_mad_u16 v1, v6, v13, v1
+; GFX8-NEXT: v_bfe_u32 v15, v0, 16, 4
+; GFX8-NEXT: v_mad_u16 v1, v7, v14, v1
+; GFX8-NEXT: v_bfe_u32 v9, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 20, 4
+; GFX8-NEXT: v_mad_u16 v1, v8, v15, v1
+; GFX8-NEXT: v_bfe_u32 v10, v4, 24, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 24, 4
+; GFX8-NEXT: v_mad_u16 v1, v9, v16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 28, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
+; GFX8-NEXT: v_mad_u16 v1, v10, v17, v1
+; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
+; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc16_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-NEXT: global_load_ushort v5, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s7, s6, 0x40018
-; GFX9-NEXT: s_lshr_b32 s12, s6, 28
-; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s12
-; GFX9-NEXT: s_bfe_u32 s1, s0, 0x40018
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40010
-; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40014
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, s1, v1
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s13, s14
-; GFX9-NEXT: s_bfe_u32 s15, s6, 0x40008
-; GFX9-NEXT: s_bfe_u32 s16, s6, 0x4000c
-; GFX9-NEXT: s_and_b32 s17, s6, 15
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s15, s16
-; GFX9-NEXT: s_bfe_u32 s6, s6, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s17, s6
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c
-; GFX9-NEXT: s_and_b32 s11, s0, 15
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s11, s0
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s5, s8
-; GFX9-NEXT: v_pk_mul_lo_u16 v4, s0, v4
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, s4, v2
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s9, s10
-; GFX9-NEXT: v_pk_mul_lo_u16 v3, s4, v3
+; GFX9-NEXT: global_load_dword v3, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v4, v0, s[6:7]
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_bfe_u32 v0, v3, 24, 4
+; GFX9-NEXT: v_bfe_u32 v6, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 8, 4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v5, v4, v5
-; GFX9-NEXT: v_add_u32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_sdwa v4, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
-; GFX9-NEXT: v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-NEXT: v_bfe_u32 v11, v4, 24, 4
+; GFX9-NEXT: v_bfe_u32 v13, v4, 16, 4
+; GFX9-NEXT: v_bfe_u32 v15, v4, 8, 4
+; GFX9-NEXT: v_and_b32_e32 v17, 15, v4
+; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v3
+; GFX9-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX9-NEXT: v_and_b32_e32 v6, v2, v6
+; GFX9-NEXT: v_bfe_u32 v9, v3, 12, 4
+; GFX9-NEXT: v_and_b32_e32 v8, v2, v8
+; GFX9-NEXT: v_bfe_u32 v3, v3, 4, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 28, v4
+; GFX9-NEXT: v_bfe_u32 v14, v4, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v4, 12, 4
+; GFX9-NEXT: v_bfe_u32 v4, v4, 4, 4
+; GFX9-NEXT: v_and_b32_e32 v17, v2, v17
+; GFX9-NEXT: v_and_b32_e32 v11, v2, v11
+; GFX9-NEXT: v_and_b32_e32 v13, v2, v13
+; GFX9-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX9-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v17
+; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v4
+; GFX9-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v5, v14, 16, v13
+; GFX9-NEXT: v_lshl_or_b32 v7, v16, 16, v15
+; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v6, v5
+; GFX9-NEXT: v_pk_mul_lo_u16 v5, v8, v7
+; GFX9-NEXT: v_lshl_or_b32 v10, v12, 16, v11
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v10
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v4, v2, v4
+; GFX9-NEXT: v_add_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v5
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v3
+; GFX9-NEXT: v_add_u16_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v0
+; GFX9-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0
-; GFX9-DL-NEXT: global_load_ushort v5, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s7, s6, 0x40018
-; GFX9-DL-NEXT: s_lshr_b32 s12, s6, 28
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s7, s7, s12
-; GFX9-DL-NEXT: s_bfe_u32 s1, s0, 0x40018
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40014
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, s1, v1
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s13, s14
-; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x4000c
-; GFX9-DL-NEXT: s_and_b32 s17, s6, 15
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s15, s16
-; GFX9-DL-NEXT: s_bfe_u32 s6, s6, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s17, s6
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x4000c
-; GFX9-DL-NEXT: s_and_b32 s11, s0, 15
-; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x40004
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s11, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s4, s5, s8
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, s0, v4
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, s4, v2
-; GFX9-DL-NEXT: s_pack_ll_b32_b16 s4, s9, s10
-; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, s4, v3
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v4, v0, s[6:7]
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_u32 v0, v3, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v3, 8, 4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_bfe_u32 v11, v4, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v4, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v4, 8, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v4
+; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v3
+; GFX9-DL-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX9-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v6, v2, v6
+; GFX9-DL-NEXT: v_bfe_u32 v9, v3, 12, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v8, v2, v8
+; GFX9-DL-NEXT: v_bfe_u32 v3, v3, 4, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 28, v4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v4, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v4, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v4, v4, 4, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v17, v2, v17
+; GFX9-DL-NEXT: v_and_b32_e32 v11, v2, v11
+; GFX9-DL-NEXT: v_and_b32_e32 v13, v2, v13
+; GFX9-DL-NEXT: v_and_b32_e32 v15, v2, v15
+; GFX9-DL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v17
+; GFX9-DL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4
+; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
+; GFX9-DL-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-DL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-DL-NEXT: v_lshl_or_b32 v5, v14, 16, v13
+; GFX9-DL-NEXT: v_lshl_or_b32 v7, v16, 16, v15
+; GFX9-DL-NEXT: v_lshl_or_b32 v8, v9, 16, v8
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v6, v5
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v8, v7
+; GFX9-DL-NEXT: v_lshl_or_b32 v10, v12, 16, v11
+; GFX9-DL-NEXT: v_pk_mul_lo_u16 v0, v0, v10
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v5, v4, v5
-; GFX9-DL-NEXT: v_add_u32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_sdwa v4, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v3, v3, v2
-; GFX9-DL-NEXT: v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add_u16_e32 v4, v2, v4
+; GFX9-DL-NEXT: v_add_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v5
+; GFX9-DL-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v3
+; GFX9-DL-NEXT: v_add_u16_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v0
+; GFX9-DL-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_bfe_u32 s3, s6, 0x40004
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40004
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, s0, s1
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x40014
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, s0, s2
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40014
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s3, s7, 28
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v7, 15, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 8, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v7
+; GFX10-DL-NEXT: v_and_b32_e32 v6, v4, v6
+; GFX10-DL-NEXT: v_bfe_u32 v19, v1, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 12, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v13, v4, v13
+; GFX10-DL-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX10-DL-NEXT: v_lshl_or_b32 v6, v15, 16, v6
+; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v12, v4, v19
+; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 16, 4
+; GFX10-DL-NEXT: v_lshl_or_b32 v10, v10, 16, v13
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v6, v7, v6
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 16, 4
+; GFX10-DL-NEXT: v_lshl_or_b32 v9, v9, 16, v12
+; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v23, 28, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v12, 16, v6
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, s0, s2
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40018
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, s0, s2
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v6, v3
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v9, v9, v10
+; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 20, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v11, v4, v11
+; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v7
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v12
+; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 24, 4
+; GFX10-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v11
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v9
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v14, v3, v9
+; GFX10-DL-NEXT: v_and_b32_e32 v9, v4, v10
+; GFX10-DL-NEXT: v_and_b32_e32 v4, v4, v5
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v6
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v14, v7
+; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v9
+; GFX10-DL-NEXT: v_lshl_or_b32 v4, v23, 16, v4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v1
+; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v3, v5
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
+; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i16>
%cvec2 = zext <8 x i4> %vec2 to <8 x i16>
@@ -2488,80 +2461,94 @@ entry:
define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_bfe_u32 s6, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s13, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s17, s5, 28
-; GFX7-NEXT: v_mov_b32_e32 v8, s13
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40008
-; GFX7-NEXT: s_and_b32 s16, s5, 15
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40004
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: s_lshr_b32 s10, s4, 28
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mul_u32_u24_e32 v4, s10, v4
-; GFX7-NEXT: v_mul_u32_u24_e32 v6, s8, v6
-; GFX7-NEXT: v_mul_u32_u24_e32 v8, s6, v8
-; GFX7-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40008
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
-; GFX7-NEXT: s_and_b32 s9, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40018
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40014
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mul_u32_u24_e32 v2, s12, v2
-; GFX7-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mul_u32_u24_e32 v3, s11, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_mul_u32_u24_e32 v5, s9, v5
-; GFX7-NEXT: v_mul_u32_u24_e32 v7, s7, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX7-NEXT: v_or_b32_e32 v4, v5, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v7, v8
-; GFX7-NEXT: v_mul_u32_u24_e32 v9, s4, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v9, v2
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: s_movk_i32 s4, 0xf00
+; GFX7-NEXT: v_mov_b32_e32 v3, 0xf00
+; GFX7-NEXT: s_movk_i32 s5, 0xf0f
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 28, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v9, 4, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 4, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 12, v2
+; GFX7-NEXT: v_bfe_u32 v1, v2, 8, 4
+; GFX7-NEXT: v_and_b32_e32 v5, 15, v2
+; GFX7-NEXT: v_bfe_u32 v7, v2, 16, 4
+; GFX7-NEXT: v_alignbit_b32 v2, v6, v2, 24
+; GFX7-NEXT: v_and_b32_e32 v6, s4, v9
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v11, 4, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v9, 4, v0
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
+; GFX7-NEXT: v_and_b32_e32 v6, v3, v9
+; GFX7-NEXT: v_bfe_u32 v10, v0, 8, 4
+; GFX7-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-NEXT: v_or_b32_e32 v3, v10, v3
+; GFX7-NEXT: v_and_b32_e32 v12, 15, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v13, 28, v0
+; GFX7-NEXT: v_or_b32_e32 v6, v12, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v4, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v2, v3, 8
-; GFX7-NEXT: v_alignbit_b32 v5, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_and_b32_e32 v2, s5, v2
+; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v15, 12, v0
+; GFX7-NEXT: v_bfe_u32 v14, v0, 16, 4
+; GFX7-NEXT: v_alignbit_b32 v0, v13, v0, 24
+; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v15
+; GFX7-NEXT: v_and_b32_e32 v0, s5, v0
+; GFX7-NEXT: v_or_b32_e32 v7, v7, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v6, 15, v1
+; GFX7-NEXT: v_and_b32_e32 v12, 15, v3
+; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX7-NEXT: v_or_b32_e32 v4, v14, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX7-NEXT: v_bfe_u32 v13, v3, 8, 4
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_mad_u32_u24 v6, v6, v12, v16
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 24, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 24, v3
+; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 4
+; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 4
+; GFX7-NEXT: v_mad_u32_u24 v6, v7, v13, v6
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v6
+; GFX7-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX7-NEXT: v_and_b32_e32 v14, 15, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v4, v10, v1
+; GFX7-NEXT: v_bfe_u32 v9, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 4
+; GFX7-NEXT: v_mad_u32_u24 v1, v8, v14, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v0
+; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 4
+; GFX7-NEXT: v_mad_u32_u24 v1, v9, v15, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v11, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -2569,291 +2556,285 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s22, -1
-; GFX8-NEXT: s_mov_b32 s23, 0xe80000
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v2, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s20, s20, s3
-; GFX8-NEXT: s_addc_u32 s21, s21, 0
-; GFX8-NEXT: s_mov_b32 s0, 0xffff
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_bfe_u32 s7, s1, 0x40004
-; GFX8-NEXT: s_bfe_u32 s9, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40004
-; GFX8-NEXT: s_and_b32 s15, s2, 15
-; GFX8-NEXT: s_bfe_u32 s16, s2, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s3, s1, 0x40014
-; GFX8-NEXT: s_lshr_b32 s5, s1, 28
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40014
-; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40010
-; GFX8-NEXT: s_lshr_b32 s12, s2, 28
-; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40008
-; GFX8-NEXT: s_and_b32 s8, s1, 15
-; GFX8-NEXT: v_mov_b32_e32 v4, s16
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
-; GFX8-NEXT: v_mov_b32_e32 v6, s15
-; GFX8-NEXT: v_mov_b32_e32 v7, s14
-; GFX8-NEXT: v_mov_b32_e32 v8, s7
-; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mul_u32_u24_e32 v5, s8, v6
-; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s1, 0x40018
-; GFX8-NEXT: v_mov_b32_e32 v9, s13
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40008
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_mov_b32_e32 v10, s12
-; GFX8-NEXT: v_mov_b32_e32 v11, s5
-; GFX8-NEXT: v_mov_b32_e32 v12, s11
-; GFX8-NEXT: v_mov_b32_e32 v13, s10
-; GFX8-NEXT: v_mov_b32_e32 v14, s3
-; GFX8-NEXT: v_mul_u32_u24_e32 v3, s1, v3
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX8-NEXT: v_mul_u32_u24_e32 v7, s6, v9
-; GFX8-NEXT: v_mul_u32_u24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mul_u32_u24_e32 v9, s4, v12
-; GFX8-NEXT: v_mul_u32_u24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_e32 v5, s0, v5
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: flat_load_ubyte v5, v[0:1]
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_bfe_u32 v3, v4, 20, 4
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_bfe_u32 v13, v2, 20, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 24, 4
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 28, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v15, 28, v2
+; GFX8-NEXT: v_bfe_u32 v14, v2, 24, 4
+; GFX8-NEXT: v_mul_lo_u16_sdwa v3, v3, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v12, v2, 16, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v16, v2, 8, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 12, 4
+; GFX8-NEXT: v_and_b32_e32 v11, 15, v4
+; GFX8-NEXT: v_bfe_u32 v17, v2, 12, 4
+; GFX8-NEXT: v_and_b32_e32 v18, 15, v2
+; GFX8-NEXT: v_bfe_u32 v4, v4, 4, 4
+; GFX8-NEXT: v_bfe_u32 v2, v2, 4, 4
+; GFX8-NEXT: v_mul_lo_u16_e32 v13, v7, v14
+; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_mul_lo_u16_e32 v19, v6, v12
+; GFX8-NEXT: v_mul_lo_u16_e32 v9, v9, v16
+; GFX8-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_mul_lo_u16_e32 v11, v11, v18
+; GFX8-NEXT: v_mul_lo_u16_sdwa v4, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v8, v13, v8
; GFX8-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_e32 v4, s0, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v6, v4, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v6
+; GFX8-NEXT: v_or_b32_e32 v10, v11, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX8-NEXT: v_or_b32_e32 v3, v19, v3
+; GFX8-NEXT: v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v9
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v3
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 24, v[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v8, v2
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v3, v10, v5
+; GFX8-NEXT: v_add_u16_e32 v3, v3, v4
+; GFX8-NEXT: v_add_u16_e32 v3, v3, v9
+; GFX8-NEXT: v_add_u16_e32 v2, v3, v2
+; GFX8-NEXT: v_mad_u16 v2, v6, v12, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v11
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX8-NEXT: v_mad_u16 v2, v7, v14, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_mov_b32 s0, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s4, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s12, s8, 0x40010
-; GFX9-NEXT: s_bfe_u32 s13, s8, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s8, 0x40018
-; GFX9-NEXT: s_lshr_b32 s15, s8, 28
-; GFX9-NEXT: s_and_b32 s16, s8, 15
-; GFX9-NEXT: s_bfe_u32 s17, s8, 0x40004
-; GFX9-NEXT: s_bfe_u32 s18, s8, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: s_bfe_u32 s8, s8, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s5, s1, 0x40014
-; GFX9-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-NEXT: s_bfe_u32 s6, s1, 0x40018
-; GFX9-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-NEXT: s_lshr_b32 s7, s1, 28
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: s_and_b32 s9, s1, 15
-; GFX9-NEXT: v_mov_b32_e32 v6, s16
-; GFX9-NEXT: s_bfe_u32 s10, s1, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v7, s17
-; GFX9-NEXT: s_bfe_u32 s11, s1, 0x40008
-; GFX9-NEXT: v_mov_b32_e32 v8, s18
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v9, s8
-; GFX9-NEXT: v_mul_lo_u16_e32 v2, s4, v2
-; GFX9-NEXT: v_mul_lo_u16_sdwa v3, s5, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v4, s6, v4
-; GFX9-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v6, s9, v6
-; GFX9-NEXT: v_mul_lo_u16_sdwa v7, s10, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_or_b32_sdwa v3, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v4, v6, v7
-; GFX9-NEXT: v_mul_lo_u16_e32 v8, s11, v8
-; GFX9-NEXT: v_mul_lo_u16_sdwa v9, s1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v4, s0, v4
-; GFX9-NEXT: v_or_b32_sdwa v5, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v5, v4, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 8, v5
-; GFX9-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v2, v3
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v4, v3, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_bfe_u32 v0, v1, 20, 4
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v6, v1, 24, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 28, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 28, v2
+; GFX9-NEXT: v_bfe_u32 v13, v2, 24, 4
+; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v0, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 4
+; GFX9-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 12, 4
+; GFX9-NEXT: v_and_b32_e32 v10, 15, v1
+; GFX9-NEXT: v_bfe_u32 v16, v2, 12, 4
+; GFX9-NEXT: v_and_b32_e32 v17, 15, v2
+; GFX9-NEXT: v_bfe_u32 v1, v1, 4, 4
+; GFX9-NEXT: v_bfe_u32 v2, v2, 4, 4
+; GFX9-NEXT: v_mul_lo_u16_e32 v12, v6, v13
+; GFX9-NEXT: v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v18, v5, v11
+; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v8, v8, v15
+; GFX9-NEXT: v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v10, v10, v17
+; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
+; GFX9-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX9-NEXT: v_or_b32_e32 v1, v18, v0
+; GFX9-NEXT: v_or_b32_e32 v9, v10, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v6
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_add_u16_e32 v1, v9, v4
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v8
+; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v5, v11, v0
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v7
+; GFX9-NEXT: v_mad_legacy_u16 v0, v6, v13, v0
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
+; GFX9-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_bfe_u32 s4, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s12, s8, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s13, s8, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s8, 0x40018
-; GFX9-DL-NEXT: s_lshr_b32 s15, s8, 28
-; GFX9-DL-NEXT: s_and_b32 s16, s8, 15
-; GFX9-DL-NEXT: s_bfe_u32 s17, s8, 0x40004
-; GFX9-DL-NEXT: s_bfe_u32 s18, s8, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-DL-NEXT: s_bfe_u32 s8, s8, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s5, s1, 0x40014
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13
-; GFX9-DL-NEXT: s_bfe_u32 s6, s1, 0x40018
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 28
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: s_and_b32 s9, s1, 15
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s16
-; GFX9-DL-NEXT: s_bfe_u32 s10, s1, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s17
-; GFX9-DL-NEXT: s_bfe_u32 s11, s1, 0x40008
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s18
-; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-DL-NEXT: v_mov_b32_e32 v9, s8
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v2, s4, v2
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v3, s5, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, s6, v4
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v6, s9, v6
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, s10, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-DL-NEXT: v_or_b32_sdwa v3, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v4, v6, v7
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, s11, v8
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v9, s1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v4
-; GFX9-DL-NEXT: v_or_b32_sdwa v5, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v5, v4, v5
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v5
-; GFX9-DL-NEXT: v_and_b32_e32 v2, s0, v2
-; GFX9-DL-NEXT: v_or_b32_e32 v3, v2, v3
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[2:3]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 20, 4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 24, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 28, v1
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v2
+; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v0, v0, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v1
+; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 12, 4
+; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v2
+; GFX9-DL-NEXT: v_bfe_u32 v1, v1, 4, 4
+; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 4, 4
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v12, v6, v13
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v5, v11
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, v8, v15
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v10, v10, v17
+; GFX9-DL-NEXT: v_or_b32_e32 v7, v12, v7
+; GFX9-DL-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX9-DL-NEXT: v_or_b32_e32 v1, v18, v0
+; GFX9-DL-NEXT: v_or_b32_e32 v9, v10, v2
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX9-DL-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v6
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v3
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v9, v4
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v8
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v5, v11, v0
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v10
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
+; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v6, v13, v0
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
+; GFX9-DL-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX10-DL-NEXT: s_mov_b32 s14, -1
-; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT: v_mov_b32_e32 v19, 0
+; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX10-DL-NEXT: s_mov_b32 s10, -1
+; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
+; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
+; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40004
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, s1, s3
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
-; GFX10-DL-NEXT: s_bfe_u32 s8, s6, 0x4000c
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v3, s0, s1
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s8, s3
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v2, 8, v2
-; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x40008
-; GFX10-DL-NEXT: s_mov_b32 s1, 0xffff
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s2, s0
-; GFX10-DL-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4
-; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s8, s7, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: v_and_b32_e32 v2, s1, v2
-; GFX10-DL-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s2, s8
-; GFX10-DL-NEXT: s_bfe_u32 s3, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s9, s7, 0x40010
-; GFX10-DL-NEXT: s_lshr_b32 s6, s6, 28
-; GFX10-DL-NEXT: v_or_b32_e32 v3, v2, v3
-; GFX10-DL-NEXT: s_lshr_b32 s2, s7, 28
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s0, s9
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v6, s6, s2
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
-; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x40018
-; GFX10-DL-NEXT: v_mul_lo_u16_e64 v11, s3, s0
-; GFX10-DL-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX10-DL-NEXT: v_lshlrev_b16_e64 v6, 8, v6
-; GFX10-DL-NEXT: v_and_b32_e32 v4, s1, v4
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v3, v19, s[0:1]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 8, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 28, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v2
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v9, v10
+; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 16, 4
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v8, v8, v13
+; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 24, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v1
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v9, 8, v9
+; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 4, 4
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v7, v7, v14
+; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 20, 4
+; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v23, v2, 16, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v1, v1, v15
+; GFX10-DL-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v0, v10
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v10, v6, v13
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v7, 8, v7
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, v11, v2
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-NEXT: v_mul_lo_u16_e64 v11, v5, v23
+; GFX10-DL-NEXT: v_or_b32_e32 v7, v10, v7
+; GFX10-DL-NEXT: v_lshlrev_b16_e64 v9, 8, v9
+; GFX10-DL-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX10-DL-NEXT: v_or_b32_e32 v2, v11, v9
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v10
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1
-; GFX10-DL-NEXT: v_or_b32_sdwa v2, v11, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v7
-; GFX10-DL-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v4
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v1, v3
+; GFX10-DL-NEXT: v_or_b32_sdwa v1, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v9, v3, v10
+; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v9, v8
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v2
+; GFX10-DL-NEXT: v_mad_u16 v0, v5, v23, v0
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v7
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v13, v0
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v1
+; GFX10-DL-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i8>
%cvec2 = zext <8 x i4> %vec2 to <8 x i8>
@@ -2886,54 +2867,53 @@ entry:
define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc4_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX7-NEXT: s_mov_b32 s22, -1
-; GFX7-NEXT: s_mov_b32 s23, 0xe8f000
-; GFX7-NEXT: s_add_u32 s20, s20, s3
+; GFX7-NEXT: s_mov_b32 s14, -1
+; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_addc_u32 s21, s21, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s4, 28
-; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004
-; GFX7-NEXT: s_lshr_b32 s13, s5, 28
-; GFX7-NEXT: s_and_b32 s5, s5, 15
-; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004
-; GFX7-NEXT: s_and_b32 s4, s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mov_b32_e32 v2, s19
-; GFX7-NEXT: v_mov_b32_e32 v3, s18
-; GFX7-NEXT: v_mov_b32_e32 v4, s17
-; GFX7-NEXT: v_mov_b32_e32 v5, s16
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
+; GFX7-NEXT: s_addc_u32 s13, s13, 0
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
+; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
+; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
+; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
@@ -2942,223 +2922,238 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT: s_mov_b32 s18, -1
-; GFX8-NEXT: s_mov_b32 s19, 0xe80000
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT: s_add_u32 s16, s16, s3
-; GFX8-NEXT: s_addc_u32 s17, s17, 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s8, s0, 15
-; GFX8-NEXT: s_and_b32 s15, s1, 15
-; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v4, s15
-; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
-; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
-; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
-; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
-; GFX8-NEXT: s_lshr_b32 s9, s1, 28
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: s_lshr_b32 s2, s0, 28
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3
-; GFX8-NEXT: v_and_b32_e32 v3, 15, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, s12
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s11, 0xe80000
+; GFX8-NEXT: s_add_u32 s8, s8, s3
+; GFX8-NEXT: s_addc_u32 s9, s9, 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 4, 4
+; GFX8-NEXT: v_bfe_u32 v6, v4, 8, 4
+; GFX8-NEXT: v_bfe_u32 v7, v4, 12, 4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2
-; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s9
-; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_and_b32_e32 v11, 15, v0
+; GFX8-NEXT: v_bfe_u32 v12, v0, 4, 4
+; GFX8-NEXT: v_bfe_u32 v13, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v14, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v8, v4, 16, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v9, v4, 20, 4
+; GFX8-NEXT: v_bfe_u32 v10, v4, 24, 4
+; GFX8-NEXT: v_bfe_u32 v16, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v17, v0, 24, 4
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 28, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
+; GFX8-NEXT: v_mul_u32_u24_e32 v0, v4, v0
+; GFX8-NEXT: v_mul_u32_u24_e32 v4, v10, v17
+; GFX8-NEXT: flat_load_ubyte v10, v[2:3]
+; GFX8-NEXT: v_mul_u32_u24_e32 v1, v1, v11
+; GFX8-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX8-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX8-NEXT: v_mul_u32_u24_e32 v7, v7, v14
+; GFX8-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX8-NEXT: v_mul_u32_u24_e32 v9, v9, v16
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v10
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v5
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v6
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v7
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v8
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v9
+; GFX8-NEXT: v_add_u16_e32 v1, v1, v4
+; GFX8-NEXT: v_add_u16_e32 v0, v1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udot8_acc4_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s22, -1
-; GFX9-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-NEXT: s_add_u32 s20, s20, s3
+; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s10, s0, 15
-; GFX9-NEXT: s_and_b32 s17, s1, 15
-; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-NEXT: v_bfe_u32 v4, v2, 4, 4
+; GFX9-NEXT: v_bfe_u32 v11, v3, 4, 4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v12, v3, 8, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v13, v3, 12, 4
+; GFX9-NEXT: v_bfe_u32 v7, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v14, v3, 16, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v9, v2, 24, 4
+; GFX9-NEXT: v_bfe_u32 v15, v3, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v3, 24, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX9-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX9-NEXT: v_mul_u32_u24_e32 v3, v9, v16
+; GFX9-NEXT: global_load_ubyte v9, v1, s[2:3]
+; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v10
+; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
+; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v9
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v4
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v5
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v6
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v3
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc4_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; GFX9-DL-NEXT: s_mov_b32 s22, -1
-; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000
-; GFX9-DL-NEXT: s_add_u32 s20, s20, s3
+; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX9-DL-NEXT: s_mov_b32 s10, -1
+; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
+; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_and_b32 s10, s0, 15
-; GFX9-DL-NEXT: s_and_b32 s17, s1, 15
-; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
-; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008
-; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
-; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
-; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
-; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
-; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
-; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
-; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
-; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
-; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11
-; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
-; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
+; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 4, 4
+; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 4, 4
+; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 12, 4
+; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 16, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 24, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 24, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v3, v9, v16
+; GFX9-DL-NEXT: global_load_ubyte v9, v1, s[2:3]
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v0, v0, v10
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v11
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v12
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v13
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v14
+; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v15
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v9
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v4
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v5
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v6
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v3
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc4_vecMul:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
-; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
-; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_and_b32 s0, s6, 15
-; GFX10-DL-NEXT: s_and_b32 s1, s7, 15
-; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
+; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DL-NEXT: v_and_b32_e32 v4, 15, v1
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v2
+; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
+; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 8, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v7
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008
-; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c
-; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s1, s3
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s2, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v4, v3
+; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 12, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v8
+; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 16, 4
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v6
+; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v7
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 20, 4
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v5
+; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v8
+; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 24, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v1
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v4
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v11, v8
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v6
+; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v2
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v5
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v4
+; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018
-; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28
-; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
-; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
-; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) {
entry:
- %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
- %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
+ %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
+ %gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
+ %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%mul = mul <8 x i4> %vec1, %vec2
%mul0 = extractelement <8 x i4> %mul, i64 0
@@ -3190,45 +3185,44 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_and_b32_e32 v1, 15, v2
+; GFX7-NEXT: v_bfe_u32 v3, v2, 4, 4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v9, 15, v0
+; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 4
+; GFX7-NEXT: v_bfe_u32 v5, v2, 12, 4
+; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 4
+; GFX7-NEXT: v_bfe_u32 v7, v2, 20, 4
+; GFX7-NEXT: v_bfe_u32 v8, v2, 24, 4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX7-NEXT: v_bfe_u32 v10, v0, 4, 4
+; GFX7-NEXT: v_bfe_u32 v11, v0, 8, 4
+; GFX7-NEXT: v_bfe_u32 v12, v0, 12, 4
+; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 4
+; GFX7-NEXT: v_bfe_u32 v14, v0, 20, 4
+; GFX7-NEXT: v_bfe_u32 v15, v0, 24, 4
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s6, s4, 15
-; GFX7-NEXT: s_and_b32 s7, s5, 15
-; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40004
-; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40008
-; GFX7-NEXT: s_bfe_u32 s12, s4, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s14, s4, 0x40010
-; GFX7-NEXT: s_bfe_u32 s16, s4, 0x40014
-; GFX7-NEXT: s_bfe_u32 s18, s4, 0x40018
-; GFX7-NEXT: s_lshr_b32 s4, s4, 28
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT: s_bfe_u32 s9, s5, 0x40004
-; GFX7-NEXT: s_bfe_u32 s11, s5, 0x40008
-; GFX7-NEXT: s_bfe_u32 s13, s5, 0x4000c
-; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40010
-; GFX7-NEXT: s_bfe_u32 s17, s5, 0x40014
-; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40018
-; GFX7-NEXT: s_lshr_b32 s5, s5, 28
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s14
-; GFX7-NEXT: v_mad_u32_u24 v0, s15, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s16
-; GFX7-NEXT: v_mad_u32_u24 v0, s17, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s18
-; GFX7-NEXT: v_mad_u32_u24 v0, s19, v1, v0
+; GFX7-NEXT: v_mad_u32_u24 v1, v9, v1, s4
+; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
+; GFX7-NEXT: v_mad_u32_u24 v0, v10, v3, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v11, v4, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v12, v5, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v13, v6, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v14, v7, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v15, v8, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -3236,44 +3230,44 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 4, 4
+; GFX8-NEXT: v_bfe_u32 v6, v3, 8, 4
+; GFX8-NEXT: v_bfe_u32 v8, v3, 12, 4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX8-NEXT: v_bfe_u32 v5, v0, 4, 4
+; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
+; GFX8-NEXT: v_bfe_u32 v9, v0, 12, 4
+; GFX8-NEXT: v_bfe_u32 v10, v3, 16, 4
+; GFX8-NEXT: v_bfe_u32 v11, v0, 16, 4
+; GFX8-NEXT: v_bfe_u32 v12, v3, 20, 4
+; GFX8-NEXT: v_bfe_u32 v14, v3, 24, 4
+; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
+; GFX8-NEXT: v_bfe_u32 v15, v0, 24, 4
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 28, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s4, s2, 15
-; GFX8-NEXT: s_and_b32 s5, s3, 15
-; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40004
-; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40008
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s12, s2, 0x40010
-; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40014
-; GFX8-NEXT: s_bfe_u32 s16, s2, 0x40018
-; GFX8-NEXT: s_lshr_b32 s2, s2, 28
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s18
-; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1
-; GFX8-NEXT: s_bfe_u32 s7, s3, 0x40004
-; GFX8-NEXT: s_bfe_u32 s9, s3, 0x40008
-; GFX8-NEXT: s_bfe_u32 s11, s3, 0x4000c
-; GFX8-NEXT: s_bfe_u32 s13, s3, 0x40010
-; GFX8-NEXT: s_bfe_u32 s15, s3, 0x40014
-; GFX8-NEXT: s_bfe_u32 s17, s3, 0x40018
-; GFX8-NEXT: s_lshr_b32 s3, s3, 28
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mad_u32_u24 v0, s3, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mad_u32_u24 v0, s13, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s14
-; GFX8-NEXT: v_mad_u32_u24 v0, s15, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mad_u32_u24 v2, s17, v1, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, s2
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v1
+; GFX8-NEXT: v_mad_u32_u24 v0, v5, v4, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v7, v6, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v9, v8, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v11, v10, v0
+; GFX8-NEXT: v_mad_u32_u24 v0, v13, v12, v0
+; GFX8-NEXT: v_mad_u32_u24 v2, v15, v14, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -3283,45 +3277,43 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_e32 v3, 15, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX9-NEXT: v_bfe_u32 v5, v1, 4, 4
+; GFX9-NEXT: v_bfe_u32 v6, v2, 4, 4
+; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 12, 4
+; GFX9-NEXT: v_bfe_u32 v10, v2, 12, 4
+; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 4
+; GFX9-NEXT: v_bfe_u32 v12, v2, 16, 4
+; GFX9-NEXT: v_bfe_u32 v13, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v15, v1, 24, 4
+; GFX9-NEXT: v_bfe_u32 v14, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 24, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 28, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v3, v4, v3
+; GFX9-NEXT: v_mul_u32_u24_e32 v1, v2, v1
+; GFX9-NEXT: v_mul_u32_u24_e32 v4, v6, v5
+; GFX9-NEXT: v_mul_u32_u24_e32 v5, v8, v7
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s4, s0, 15
-; GFX9-NEXT: s_and_b32 s5, s1, 15
-; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40004
-; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
-; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s12, s0, 0x40010
-; GFX9-NEXT: s_bfe_u32 s14, s0, 0x40014
-; GFX9-NEXT: s_bfe_u32 s16, s0, 0x40018
-; GFX9-NEXT: s_lshr_b32 s0, s0, 28
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: v_mad_u32_u24 v1, s5, v1, v2
-; GFX9-NEXT: s_bfe_u32 s7, s1, 0x40004
-; GFX9-NEXT: s_bfe_u32 s9, s1, 0x40008
-; GFX9-NEXT: s_bfe_u32 s11, s1, 0x4000c
-; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40010
-; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40014
-; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40018
-; GFX9-NEXT: s_lshr_b32 s1, s1, 28
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_mad_u32_u24 v1, s7, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_mad_u32_u24 v1, s13, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_mad_u32_u24 v1, s15, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mad_u32_u24 v1, s17, v2, v1
+; GFX9-NEXT: v_add3_u32 v1, v3, s0, v1
+; GFX9-NEXT: v_mul_u32_u24_e32 v6, v10, v9
+; GFX9-NEXT: v_mul_u32_u24_e32 v7, v12, v11
+; GFX9-NEXT: v_add3_u32 v1, v1, v4, v5
+; GFX9-NEXT: v_mul_u32_u24_e32 v8, v14, v13
+; GFX9-NEXT: v_mul_u32_u24_e32 v9, v16, v15
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v7
+; GFX9-NEXT: v_add3_u32 v1, v1, v8, v9
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
@@ -3329,38 +3321,40 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s1, v1, v2
-; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
+; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
+; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v3, v2, s0
+; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_variant1:
; GFX10-DL: ; %bb.0: ; %entry
-; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
-; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s1, s0, v0
-; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: s_clause 0x1
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
+; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s2
+; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm
i32 addrspace(1)* %v2addr,
i32 addrspace(1)* %dst) {
entry:
- %v1 = load i32, i32 addrspace(1)* %v1addr, align 4
- %v2 = load i32, i32 addrspace(1)* %v2addr, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr i32, i32 addrspace(1)* %v1addr, i32 %idx
+ %v1 = load i32, i32 addrspace(1)* %gep1, align 4
+ %gep2 = getelementptr i32, i32 addrspace(1)* %v2addr, i32 %idx
+ %v2 = load i32, i32 addrspace(1)* %gep2, align 4
%and = and i32 %v1, 15
%and1 = and i32 %v2, 15
%mul1 = mul nuw nsw i32 %and1, %and
@@ -3417,3 +3411,5 @@ entry:
store i32 %add8, i32 addrspace(1)* %dst, align 4
ret void
}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
index 561037e4b24e..c9a2b7fafd6f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
@@ -366,13 +366,13 @@ main_body:
;CHECK-LABEL: {{^}}buffer_load_ubyte_mul_bitcast:
;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ubyte_mul_bitcast(<4 x i32> inreg %rsrc) {
+define amdgpu_ps float @buffer_load_ubyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body:
- %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+ %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = zext i8 %tmp to i32
%tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float
@@ -381,13 +381,13 @@ main_body:
;CHECK-LABEL: {{^}}buffer_load_ushort_mul_bitcast:
;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ushort_mul_bitcast(<4 x i32> inreg %rsrc) {
+define amdgpu_ps float @buffer_load_ushort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body:
- %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+ %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = zext i16 %tmp to i32
%tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float
@@ -396,13 +396,13 @@ main_body:
;CHECK-LABEL: {{^}}buffer_load_sbyte_mul_bitcast:
;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sbyte_mul_bitcast(<4 x i32> inreg %rsrc) {
+define amdgpu_ps float @buffer_load_sbyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body:
- %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+ %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = sext i8 %tmp to i32
%tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float
@@ -411,13 +411,13 @@ main_body:
;CHECK-LABEL: {{^}}buffer_load_sshort_mul_bitcast:
;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sshort_mul_bitcast(<4 x i32> inreg %rsrc) {
+define amdgpu_ps float @buffer_load_sshort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body:
- %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+ %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = sext i16 %tmp to i32
%tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 7850d92bf473..6e96661b7f38 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -45,8 +45,11 @@ define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
; SDWA: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) #0 {
- %a = load i32, i32 addrspace(1)* %in1, align 4
- %b = load i32, i32 addrspace(1)* %in2, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep1 = getelementptr i32, i32 addrspace(1)* %in1, i32 %idx
+ %gep2 = getelementptr i32, i32 addrspace(1)* %in2, i32 %idx
+ %a = load i32, i32 addrspace(1)* %gep1, align 4
+ %b = load i32, i32 addrspace(1)* %gep2, align 4
%shra = lshr i32 %a, 16
%shrb = lshr i32 %b, 16
%mul = mul i32 %shra, %shrb
@@ -55,16 +58,19 @@ define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
}
; GCN-LABEL: {{^}}mul_i16:
-; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10: v_mul_lo_u16_e64 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; SDWA-NOT: v_mul_u32_u24_sdwa
define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) #0 {
entry:
- %a = load i16, i16 addrspace(1)* %ina, align 4
- %b = load i16, i16 addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr i16, i16 addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr i16, i16 addrspace(1)* %inb, i32 %idx
+ %a = load i16, i16 addrspace(1)* %gepa, align 4
+ %b = load i16, i16 addrspace(1)* %gepb, align 4
%mul = mul i16 %a, %b
store i16 %mul, i16 addrspace(1)* %out, align 4
ret void
@@ -73,21 +79,24 @@ entry:
; GCN-LABEL: {{^}}mul_v2i16:
; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]]
+; NOSDWA: v_mul_lo_u16_e32 v[[DST_MUL:[0-9]+]], v[[DST1]], v[[DST0]]
; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]]
; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
entry:
- %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
- %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %inb, i32 %idx
+ %a = load <2 x i16>, <2 x i16> addrspace(1)* %gepa, align 4
+ %b = load <2 x i16>, <2 x i16> addrspace(1)* %gepb, align 4
%mul = mul <2 x i16> %a, %b
store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
ret void
@@ -96,25 +105,28 @@ entry:
; GCN-LABEL: {{^}}mul_v4i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) #0 {
entry:
- %a = load <4 x i16>, <4 x i16> addrspace(1)* %ina, align 4
- %b = load <4 x i16>, <4 x i16> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %inb, i32 %idx
+ %a = load <4 x i16>, <4 x i16> addrspace(1)* %gepa, align 4
+ %b = load <4 x i16>, <4 x i16> addrspace(1)* %gepb, align 4
%mul = mul <4 x i16> %a, %b
store <4 x i16> %mul, <4 x i16> addrspace(1)* %out, align 4
ret void
@@ -123,23 +135,23 @@ entry:
; GCN-LABEL: {{^}}mul_v8i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL4:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL4:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]]
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]]
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
+; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -148,8 +160,11 @@ entry:
define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) #0 {
entry:
- %a = load <8 x i16>, <8 x i16> addrspace(1)* %ina, align 4
- %b = load <8 x i16>, <8 x i16> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <8 x i16>, <8 x i16> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <8 x i16>, <8 x i16> addrspace(1)* %inb, i32 %idx
+ %a = load <8 x i16>, <8 x i16> addrspace(1)* %gepa, align 4
+ %b = load <8 x i16>, <8 x i16> addrspace(1)* %gepb, align 4
%mul = mul <8 x i16> %a, %b
store <8 x i16> %mul, <8 x i16> addrspace(1)* %out, align 4
ret void
@@ -250,16 +265,19 @@ entry:
}
; GCN-LABEL: {{^}}mul_i8:
-; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10: v_mul_lo_u16_e64 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; SDWA-NOT: v_mul_u32_u24_sdwa
define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) #0 {
entry:
- %a = load i8, i8 addrspace(1)* %ina, align 4
- %b = load i8, i8 addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr i8, i8 addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr i8, i8 addrspace(1)* %inb, i32 %idx
+ %a = load i8, i8 addrspace(1)* %gepa, align 4
+ %b = load i8, i8 addrspace(1)* %gepb, align 4
%mul = mul i8 %a, %b
store i8 %mul, i8 addrspace(1)* %out, align 4
ret void
@@ -268,12 +286,12 @@ entry:
; GCN-LABEL: {{^}}mul_v2i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -287,8 +305,11 @@ entry:
; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) #0 {
entry:
- %a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4
- %b = load <2 x i8>, <2 x i8> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %inb, i32 %idx
+ %a = load <2 x i8>, <2 x i8> addrspace(1)* %gepa, align 4
+ %b = load <2 x i8>, <2 x i8> addrspace(1)* %gepb, align 4
%mul = mul <2 x i8> %a, %b
store <2 x i8> %mul, <2 x i8> addrspace(1)* %out, align 4
ret void
@@ -297,14 +318,14 @@ entry:
; GCN-LABEL: {{^}}mul_v4i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa
@@ -317,8 +338,11 @@ entry:
define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) #0 {
entry:
- %a = load <4 x i8>, <4 x i8> addrspace(1)* %ina, align 4
- %b = load <4 x i8>, <4 x i8> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %inb, i32 %idx
+ %a = load <4 x i8>, <4 x i8> addrspace(1)* %gepa, align 4
+ %b = load <4 x i8>, <4 x i8> addrspace(1)* %gepb, align 4
%mul = mul <4 x i8> %a, %b
store <4 x i8> %mul, <4 x i8> addrspace(1)* %out, align 4
ret void
@@ -327,17 +351,17 @@ entry:
; GCN-LABEL: {{^}}mul_v8i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
-; VI-DAG: v_mul_u32_u24_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
+; VI-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa
@@ -357,8 +381,11 @@ entry:
define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) #0 {
entry:
- %a = load <8 x i8>, <8 x i8> addrspace(1)* %ina, align 4
- %b = load <8 x i8>, <8 x i8> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %inb, i32 %idx
+ %a = load <8 x i8>, <8 x i8> addrspace(1)* %gepa, align 4
+ %b = load <8 x i8>, <8 x i8> addrspace(1)* %gepb, align 4
%mul = mul <8 x i8> %a, %b
store <8 x i8> %mul, <8 x i8> addrspace(1)* %out, align 4
ret void
@@ -412,9 +439,8 @@ entry:
; GCN-LABEL: {{^}}immediate_mul_v2i16:
; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mov_b32_e32 v[[M321:[0-9]+]], 0x141
-; VI-DAG: v_mov_b32_e32 v[[M123:[0-9]+]], 0x7b
-; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M123]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, 0x7b, v{{[0-9]+}}
+; VI-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9: s_mov_b32 s[[IMM:[0-9]+]], 0x141007b
; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, s[[IMM]]
@@ -423,7 +449,9 @@ entry:
define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
entry:
- %a = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %idx
+ %a = load <2 x i16>, <2 x i16> addrspace(1)* %gep, align 4
%mul = mul <2 x i16> %a, <i16 123, i16 321>
store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
ret void
@@ -433,20 +461,23 @@ entry:
; GCN-LABEL: {{^}}mulmul_v2i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
entry:
- %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
- %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
+ %idx = call i32 @llvm.amdgcn.workitem.id.x()
+ %gepa = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %ina, i32 %idx
+ %gepb = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %inb, i32 %idx
+ %a = load <2 x i16>, <2 x i16> addrspace(1)* %gepa, align 4
+ %b = load <2 x i16>, <2 x i16> addrspace(1)* %gepb, align 4
%mul = mul <2 x i16> %a, %b
%mul2 = mul <2 x i16> %mul, %b
store <2 x i16> %mul2, <2 x i16> addrspace(1)* %out, align 4
@@ -565,4 +596,6 @@ bb11: ; preds = %bb10, %bb2
br label %bb1
}
+declare i32 @llvm.amdgcn.workitem.id.x()
+
attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
More information about the llvm-commits
mailing list