[llvm] f759275 - [AMDGPU] Regenerate sdwa-peephole.ll
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sun Mar 12 07:04:26 PDT 2023
Author: Simon Pilgrim
Date: 2023-03-12T13:50:25Z
New Revision: f759275c1c8ed91f19a6b8db228115c7f75d460b
URL: https://github.com/llvm/llvm-project/commit/f759275c1c8ed91f19a6b8db228115c7f75d460b
DIFF: https://github.com/llvm/llvm-project/commit/f759275c1c8ed91f19a6b8db228115c7f75d460b.diff
LOG: [AMDGPU] Regenerate sdwa-peephole.ll
Added:
Modified:
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 256fa6b2a6a7..d5a1d8f649ad 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1,18 +1,60 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=NOSDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,GFX89,SDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_10,SDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX9_10,SDWA,GCN %s
-
-; GCN-LABEL: {{^}}add_shr_i32:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
-; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
-
-; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9: v_add_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10: v_add_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOSDWA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SDWA,GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SDWA,GFX9_10,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SDWA,GFX9_10,GFX10 %s
define amdgpu_kernel void @add_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; NOSDWA-LABEL: add_shr_i32:
+; NOSDWA: ; %bb.0:
+; NOSDWA-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s2
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s3
+; NOSDWA-NEXT: flat_load_dword v2, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, v2, v3
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: add_shr_i32:
+; GFX89: ; %bb.0:
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s2
+; GFX89-NEXT: v_mov_b32_e32 v1, s3
+; GFX89-NEXT: flat_load_dword v2, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s0
+; GFX89-NEXT: v_mov_b32_e32 v1, s1
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_add_u32_sdwa v2, vcc, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_shr_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u32_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: add_shr_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_nc_u32_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
%a = load i32, ptr addrspace(1) %in, align 4
%shr = lshr i32 %a, 16
%add = add i32 %a, %shr
@@ -20,15 +62,57 @@ define amdgpu_kernel void @add_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %
ret void
}
-; GCN-LABEL: {{^}}sub_shr_i32:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_subrev_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
-; NOSDWA-NOT: v_subrev_{{(_co)?}}_u32_sdwa
-
-; VI: v_subrev_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9: v_sub_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10: v_sub_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
define amdgpu_kernel void @sub_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; NOSDWA-LABEL: sub_shr_i32:
+; NOSDWA: ; %bb.0:
+; NOSDWA-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s2
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s3
+; NOSDWA-NEXT: flat_load_dword v2, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; NOSDWA-NEXT: v_subrev_u32_e32 v2, vcc, v2, v3
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: sub_shr_i32:
+; GFX89: ; %bb.0:
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s2
+; GFX89-NEXT: v_mov_b32_e32 v1, s3
+; GFX89-NEXT: flat_load_dword v2, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s0
+; GFX89-NEXT: v_mov_b32_e32 v1, s1
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_subrev_u32_sdwa v2, vcc, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: sub_shr_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_sub_u32_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: sub_shr_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
%a = load i32, ptr addrspace(1) %in, align 4
%shr = lshr i32 %a, 16
%sub = sub i32 %shr, %a
@@ -36,15 +120,81 @@ define amdgpu_kernel void @sub_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %
ret void
}
-; GCN-LABEL: {{^}}mul_shr_i32:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v[[DST0]], v[[DST1]]
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; SDWA: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-
define amdgpu_kernel void @mul_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #0 {
+; NOSDWA-LABEL: mul_shr_i32:
+; NOSDWA: ; %bb.0:
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dword v4, v[0:1]
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_mul_u32_u24_e32 v2, v3, v2
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_shr_i32:
+; GFX89: ; %bb.0:
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dword v4, v[0:1]
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_u32_u24_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_shr_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_shr_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gep1 = getelementptr i32, ptr addrspace(1) %in1, i32 %idx
%gep2 = getelementptr i32, ptr addrspace(1) %in2, i32 %idx
@@ -57,14 +207,78 @@ define amdgpu_kernel void @mul_shr_i32(ptr addrspace(1) %out, ptr addrspace(1) %
ret void
}
-; GCN-LABEL: {{^}}mul_i16:
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX10: v_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; SDWA-NOT: v_mul_u32_u24_sdwa
-
define amdgpu_kernel void @mul_i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 1, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_ushort v4, v[0:1]
+; NOSDWA-NEXT: flat_load_ushort v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; NOSDWA-NEXT: flat_store_short v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 1, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_ushort v4, v[0:1]
+; GFX89-NEXT: flat_load_ushort v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX89-NEXT: flat_store_short v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX9-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v1, v1, v2
+; GFX9-NEXT: global_store_short v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX10-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr i16, ptr addrspace(1) %ina, i32 %idx
@@ -76,21 +290,85 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v2i16:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v[[DST_MUL:[0-9]+]], v[[DST1]], v[[DST0]]
-; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]]
-
-; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v2i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dword v4, v[0:1]
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v4, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v3, v2
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v2i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dword v4, v[0:1]
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v3, v4, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v2i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v2i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <2 x i16>, ptr addrspace(1) %ina, i32 %idx
@@ -102,25 +380,96 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v4i16:
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
-
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v4i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; NOSDWA-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v4, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v5, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v6, v1, v3
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v7, v0, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_or_b32_e32 v1, v6, v1
+; NOSDWA-NEXT: v_or_b32_e32 v0, v7, v0
+; NOSDWA-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v4i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX89-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v4, s4
+; GFX89-NEXT: v_mov_b32_e32 v5, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v6, v1, v3
+; GFX89-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_lo_u16_e32 v3, v0, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX89-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX89-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v4i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v4i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <4 x i16>, ptr addrspace(1) %ina, i32 %idx
@@ -132,33 +481,118 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v8i16:
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL4:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]]
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]]
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
-
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v8i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v8i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 4, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v4, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; NOSDWA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; NOSDWA-NEXT: v_mov_b32_e32 v8, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v9, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v10, v3, v7
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v11, v2, v6
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v12, v1, v5
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v13, v0, v4
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v3, v7
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v2, v6
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v1, v1, v5
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v0, v0, v4
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_or_b32_e32 v3, v10, v3
+; NOSDWA-NEXT: v_or_b32_e32 v2, v11, v2
+; NOSDWA-NEXT: v_or_b32_e32 v1, v12, v1
+; NOSDWA-NEXT: v_or_b32_e32 v0, v13, v0
+; NOSDWA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v8i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 4, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v4, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX89-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX89-NEXT: v_mov_b32_e32 v8, s4
+; GFX89-NEXT: v_mov_b32_e32 v9, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v10, v3, v7
+; GFX89-NEXT: v_mul_lo_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_lo_u16_e32 v7, v2, v6
+; GFX89-NEXT: v_mul_lo_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_lo_u16_e32 v6, v1, v5
+; GFX89-NEXT: v_mul_lo_u16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_lo_u16_e32 v5, v0, v4
+; GFX89-NEXT: v_mul_lo_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_or_b32_e32 v3, v10, v3
+; GFX89-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX89-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX89-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX89-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v8i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 4, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v8i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 4, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX10-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <8 x i16>, ptr addrspace(1) %ina, i32 %idx
@@ -170,13 +604,70 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_half:
-; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_f16_sdwa
-; SDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; SDWA-NOT: v_mul_f16_sdwa
-
define amdgpu_kernel void @mul_half(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_half:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: flat_load_ushort v4, v[0:1]
+; NOSDWA-NEXT: flat_load_ushort v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_f16_e32 v2, v4, v2
+; NOSDWA-NEXT: flat_store_short v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_half:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_mov_b32_e32 v2, s0
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: flat_load_ushort v4, v[0:1]
+; GFX89-NEXT: flat_load_ushort v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_f16_e32 v2, v4, v2
+; GFX89-NEXT: flat_store_short v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_half:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX9-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_f16_e32 v1, v1, v2
+; GFX9-NEXT: global_store_short v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_half:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX10-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mul_f16_e32 v1, v1, v2
+; GFX10-NEXT: global_store_short v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load half, ptr addrspace(1) %ina, align 4
%b = load half, ptr addrspace(1) %inb, align 4
@@ -185,21 +676,78 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v2half:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mul_f16_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]]
-; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
-; NOSDWA-NOT: v_mul_f16_sdwa
-
-; VI-DAG: v_mul_f16_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_f16_e32 v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]]
-
-; GFX9_10: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v2half(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v2half:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: flat_load_dword v3, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; NOSDWA-NEXT: v_mul_f16_e32 v4, v5, v4
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; NOSDWA-NEXT: v_mul_f16_e32 v2, v3, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v4
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v2half:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_mov_b32_e32 v2, s0
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: flat_load_dword v4, v[0:1]
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v2, v4, v2
+; GFX89-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v2half:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v2half:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load <2 x half>, ptr addrspace(1) %ina, align 4
%b = load <2 x half>, ptr addrspace(1) %inb, align 4
@@ -208,23 +756,89 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v4half:
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_f16_sdwa
-
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v4half(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v4half:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; NOSDWA-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v4, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v5, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; NOSDWA-NEXT: v_mul_f16_e32 v1, v1, v3
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; NOSDWA-NEXT: v_mul_f16_e32 v0, v0, v2
+; NOSDWA-NEXT: v_mul_f16_e32 v2, v7, v6
+; NOSDWA-NEXT: v_mul_f16_e32 v3, v8, v3
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_or_b32_e32 v1, v1, v2
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v3
+; NOSDWA-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v4half:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_mov_b32_e32 v2, s0
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX89-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v4, s4
+; GFX89-NEXT: v_mov_b32_e32 v5, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_f16_sdwa v6, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX89-NEXT: v_mul_f16_sdwa v3, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX89-NEXT: v_or_b32_e32 v1, v1, v6
+; GFX89-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX89-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v4half:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v1, v1, v3
+; GFX9-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v4half:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v1, v1, v3
+; GFX10-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load <4 x half>, ptr addrspace(1) %ina, align 4
%b = load <4 x half>, ptr addrspace(1) %inb, align 4
@@ -233,29 +847,111 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v8half:
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_f16_sdwa
-
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @mul_v8half(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v8half:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v4, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v5, s7
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; NOSDWA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; NOSDWA-NEXT: v_mov_b32_e32 v8, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v9, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v10, 16, v3
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; NOSDWA-NEXT: v_mul_f16_e32 v3, v7, v3
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; NOSDWA-NEXT: v_mul_f16_e32 v2, v6, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; NOSDWA-NEXT: v_mul_f16_e32 v1, v5, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; NOSDWA-NEXT: v_mul_f16_e32 v0, v4, v0
+; NOSDWA-NEXT: v_mul_f16_e32 v4, v11, v10
+; NOSDWA-NEXT: v_mul_f16_e32 v7, v12, v7
+; NOSDWA-NEXT: v_mul_f16_e32 v6, v13, v6
+; NOSDWA-NEXT: v_mul_f16_e32 v5, v14, v5
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; NOSDWA-NEXT: v_or_b32_e32 v3, v3, v4
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v7
+; NOSDWA-NEXT: v_or_b32_e32 v1, v1, v6
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v5
+; NOSDWA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v8half:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_mov_b32_e32 v4, s0
+; GFX89-NEXT: v_mov_b32_e32 v5, s1
+; GFX89-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX89-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX89-NEXT: v_mov_b32_e32 v8, s4
+; GFX89-NEXT: v_mov_b32_e32 v9, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_f16_sdwa v10, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v3, v3, v7
+; GFX89-NEXT: v_mul_f16_sdwa v7, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v2, v2, v6
+; GFX89-NEXT: v_mul_f16_sdwa v6, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v1, v1, v5
+; GFX89-NEXT: v_mul_f16_sdwa v5, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_f16_e32 v0, v0, v4
+; GFX89-NEXT: v_or_b32_e32 v3, v3, v10
+; GFX89-NEXT: v_or_b32_e32 v2, v2, v7
+; GFX89-NEXT: v_or_b32_e32 v1, v1, v6
+; GFX89-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX89-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v8half:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v3, v3, v7
+; GFX9-NEXT: v_pk_mul_f16 v2, v2, v6
+; GFX9-NEXT: v_pk_mul_f16 v1, v1, v5
+; GFX9-NEXT: v_pk_mul_f16 v0, v0, v4
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v8half:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v3, v3, v7
+; GFX10-NEXT: v_pk_mul_f16 v2, v2, v6
+; GFX10-NEXT: v_pk_mul_f16 v1, v1, v5
+; GFX10-NEXT: v_pk_mul_f16 v0, v0, v4
+; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load <8 x half>, ptr addrspace(1) %ina, align 4
%b = load <8 x half>, ptr addrspace(1) %inb, align 4
@@ -264,14 +960,74 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_i8:
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX10: v_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; SDWA-NOT: v_mul_u32_u24_sdwa
-
define amdgpu_kernel void @mul_i8(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_i8:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s7
+; NOSDWA-NEXT: v_add_u32_e32 v1, vcc, s6, v0
+; NOSDWA-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v4, s1
+; NOSDWA-NEXT: v_add_u32_e32 v3, vcc, s0, v0
+; NOSDWA-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; NOSDWA-NEXT: flat_load_ubyte v2, v[1:2]
+; NOSDWA-NEXT: flat_load_ubyte v3, v[3:4]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v2, v3
+; NOSDWA-NEXT: flat_store_byte v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_i8:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v2, s7
+; GFX89-NEXT: v_add_u32_e32 v1, vcc, s6, v0
+; GFX89-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX89-NEXT: v_mov_b32_e32 v4, s1
+; GFX89-NEXT: v_add_u32_e32 v3, vcc, s0, v0
+; GFX89-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GFX89-NEXT: flat_load_ubyte v2, v[1:2]
+; GFX89-NEXT: flat_load_ubyte v3, v[3:4]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v2, v2, v3
+; GFX89-NEXT: flat_store_byte v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-NEXT: global_load_ubyte v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v1, v1, v2
+; GFX9-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX10-NEXT: global_load_ubyte v2, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr i8, ptr addrspace(1) %ina, i32 %idx
@@ -283,27 +1039,95 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v2i8:
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-
-; GFX9-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-
-; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-
-; GFX10: v_lshlrev_b16 v{{[0-9]+}}, 8, v
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
define amdgpu_kernel void @mul_v2i8(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v2i8:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 1, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_ushort v4, v[0:1]
+; NOSDWA-NEXT: flat_load_ushort v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v3, 8, v4
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v5, 8, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v3, v5
+; NOSDWA-NEXT: v_and_b32_e32 v2, 0xff, v2
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v3, 8, v3
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
+; NOSDWA-NEXT: flat_store_short v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v2i8:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 1, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_ushort v4, v[0:1]
+; GFX89-NEXT: flat_load_ushort v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v3, v4, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v2, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX89-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: flat_store_short v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v2i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX9-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v3, v1, v2
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_short v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v2i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_ushort v1, v0, s[6:7]
+; GFX10-NEXT: global_load_ushort v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_lshrrev_b16 v0, 8, v1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b16 v3, 8, v2
+; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v3
+; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: global_store_short v2, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <2 x i8>, ptr addrspace(1) %ina, i32 %idx
@@ -315,28 +1139,124 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v4i8:
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-
define amdgpu_kernel void @mul_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v4i8:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: flat_load_dword v4, v[0:1]
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dword v2, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 24, v4
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v6, 8, v4
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v9, 8, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v4, v6, v9
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v5, v5, v8
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v3, v7
+; NOSDWA-NEXT: v_and_b32_e32 v2, 0xff, v2
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v4, 8, v4
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v5, 8, v5
+; NOSDWA-NEXT: v_and_b32_e32 v3, 0xff, v3
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v4
+; NOSDWA-NEXT: v_or_b32_e32 v3, v3, v5
+; NOSDWA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v4i8:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dword v4, v[0:1]
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v3, v4, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v5, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX89-NEXT: v_mul_lo_u16_sdwa v6, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX89-NEXT: v_mul_lo_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v4i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v3, v1, v2
+; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_mul_lo_u16_sdwa v5, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v4i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v1
+; GFX10-NEXT: v_lshrrev_b16 v3, 8, v1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b16 v4, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: v_mul_lo_u16 v3, v3, v4
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v5
+; GFX10-NEXT: v_mul_lo_u16 v2, v6, v7
+; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
+; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: global_store_dword v2, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <4 x i8>, ptr addrspace(1) %ina, i32 %idx
@@ -348,38 +1268,172 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mul_v8i8:
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-; VI-DAG: v_mul_lo_u16_sdwa
-
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-; GFX9-DAG: v_mul_lo_u16_sdwa
-
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-; GFX10-DAG: v_mul_lo_u16
-
define amdgpu_kernel void @mul_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mul_v8i8:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; NOSDWA-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v4, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v5, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v8, 8, v0
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v10, 24, v1
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v11, 8, v1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v13, 24, v2
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v14, 8, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v16, 24, v3
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v17, 8, v3
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v11, v17
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v10, v16
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v9, v9, v15
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v8, v8, v14
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v7, v7, v13
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v6, v6, v12
+; NOSDWA-NEXT: v_and_b32_e32 v1, 0xff, v1
+; NOSDWA-NEXT: v_and_b32_e32 v0, 0xff, v0
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v3, 8, v3
+; NOSDWA-NEXT: v_and_b32_e32 v9, 0xff, v9
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v8, 8, v8
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v7, 8, v7
+; NOSDWA-NEXT: v_and_b32_e32 v6, 0xff, v6
+; NOSDWA-NEXT: v_or_b32_e32 v1, v1, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v9, v3
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v8
+; NOSDWA-NEXT: v_or_b32_e32 v3, v6, v7
+; NOSDWA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_or_b32_e32 v1, v1, v2
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v3
+; NOSDWA-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mul_v8i8:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX89-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v4, s4
+; GFX89-NEXT: v_mov_b32_e32 v5, s5
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v6, v1, v3
+; GFX89-NEXT: v_mul_lo_u16_sdwa v7, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX89-NEXT: v_mul_lo_u16_sdwa v8, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX89-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_mul_lo_u16_e32 v3, v0, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v9, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX89-NEXT: v_mul_lo_u16_sdwa v10, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX89-NEXT: v_mul_lo_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_or_b32_sdwa v2, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v3, v3, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v0, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mul_v8i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v5, v1, v3
+; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_mul_lo_u16_sdwa v7, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_mul_lo_u16_e32 v3, v0, v2
+; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_mul_lo_u16_sdwa v9, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v2, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mul_v8i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
+; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX10-NEXT: v_lshrrev_b16 v6, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX10-NEXT: v_lshrrev_b16 v9, 8, v1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-NEXT: v_lshrrev_b16 v12, 8, v2
+; GFX10-NEXT: v_lshrrev_b16 v13, 8, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 24, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GFX10-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: v_mul_lo_u16 v2, v9, v13
+; GFX10-NEXT: v_mul_lo_u16 v3, v8, v14
+; GFX10-NEXT: v_mul_lo_u16 v6, v6, v12
+; GFX10-NEXT: v_mul_lo_u16 v5, v5, v11
+; GFX10-NEXT: v_mul_lo_u16 v7, v7, v15
+; GFX10-NEXT: v_mul_lo_u16 v4, v4, v10
+; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
+; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
+; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <8 x i8>, ptr addrspace(1) %ina, i32 %idx
@@ -391,17 +1445,68 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}sitofp_v2i16_to_v2f16:
-; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_cvt_f16_i16_sdwa
-
-; SDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; SDWA-DAG: v_cvt_f16_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}} dst_sel:{{(WORD_1|DWORD)?}} dst_unused:UNUSED_PAD src0_sel:WORD_1
-
; FIXME: Should be able to avoid or
define amdgpu_kernel void @sitofp_v2i16_to_v2f16(
+; NOSDWA-LABEL: sitofp_v2i16_to_v2f16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s2
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s3
+; NOSDWA-NEXT: flat_load_dword v2, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; NOSDWA-NEXT: v_cvt_f16_i16_e32 v3, v3
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_cvt_f16_i16_e32 v2, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: sitofp_v2i16_to_v2f16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s2
+; GFX89-NEXT: v_mov_b32_e32 v1, s3
+; GFX89-NEXT: flat_load_dword v2, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s0
+; GFX89-NEXT: v_mov_b32_e32 v1, s1
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_cvt_f16_i16_sdwa v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX89-NEXT: v_cvt_f16_i16_e32 v2, v2
+; GFX89-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: sitofp_v2i16_to_v2f16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cvt_f16_i16_e32 v2, v1
+; GFX9-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: sitofp_v2i16_to_v2f16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f16_i16_e32 v2, v1
+; GFX10-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) #0 {
entry:
@@ -411,22 +1516,83 @@ entry:
ret void
}
-
-; GCN-LABEL: {{^}}mac_v2half:
-; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mac_f16_e32 v[[DST_MAC:[0-9]+]], v[[DST0]], v[[DST1]]
-; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MAC]]
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
-; NOSDWA-NOT: v_mac_f16_sdwa
-
-; VI: v_mac_f16_sdwa v[[DST_MAC:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MAC]]
-
-; GFX9_10: v_pk_mul_f16 v[[DST_MUL:[0-9]+]], v{{[0-9]+}}, v[[SRC:[0-9]+]]
-; GFX9_10: v_pk_add_f16 v{{[0-9]+}}, v[[DST_MUL]], v[[SRC]]
-
define amdgpu_kernel void @mac_v2half(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mac_v2half:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: flat_load_dword v3, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; NOSDWA-NEXT: v_mac_f16_e32 v4, v5, v4
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; NOSDWA-NEXT: v_mac_f16_e32 v2, v3, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v4
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mac_v2half:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v2, s0
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: flat_load_dword v3, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(1)
+; GFX89-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mac_f16_sdwa v4, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX89-NEXT: v_mac_f16_e32 v2, v3, v2
+; GFX89-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mac_v2half:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX9-NEXT: v_pk_add_f16 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mac_v2half:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX10-NEXT: v_pk_add_f16 v1, v1, v2
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load <2 x half>, ptr addrspace(1) %ina, align 4
%b = load <2 x half>, ptr addrspace(1) %inb, align 4
@@ -436,18 +1602,70 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}immediate_mul_v2i16:
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; VI-DAG: v_mov_b32_e32 v[[M321:[0-9]+]], 0x141
-; VI-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, 0x7b, v{{[0-9]+}}
-; VI-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-
-; GFX9: s_mov_b32 s[[IMM:[0-9]+]], 0x141007b
-; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, s[[IMM]]
-
-; GFX10: v_pk_mul_lo_u16 v{{[0-9]+}}, 0x141007b, v{{[0-9]+}}
-
define amdgpu_kernel void @immediate_mul_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; NOSDWA-LABEL: immediate_mul_v2i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s3
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: flat_load_dword v2, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, 0x7b, v2
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, 0x141, v2
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v3, v2
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: immediate_mul_v2i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX89-NEXT: v_mov_b32_e32 v3, 0x141
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s3
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: flat_load_dword v2, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s0
+; GFX89-NEXT: v_mov_b32_e32 v1, s1
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v4, 0x7b, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: immediate_mul_v2i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: s_mov_b32 s2, 0x141007b
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, s2
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: immediate_mul_v2i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, 0x141007b, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x i16>, ptr addrspace(1) %in, i32 %idx
@@ -458,20 +1676,94 @@ entry:
}
; Double use of same src - should not convert it
-; GCN-LABEL: {{^}}mulmul_v2i16:
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_mul_u32_u24_sdwa
-
-; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-
-; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
-
define amdgpu_kernel void @mulmul_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: mulmul_v2i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; NOSDWA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: flat_load_dword v3, v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v3, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v5, v5, v4
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v2, v3, v2
+; NOSDWA-NEXT: v_mul_lo_u16_e32 v3, v5, v4
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: mulmul_v2i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_add_u32_e32 v0, vcc, s6, v2
+; GFX89-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX89-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: flat_load_dword v3, v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: s_waitcnt vmcnt(1)
+; GFX89-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_mul_lo_u16_e32 v5, v3, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX89-NEXT: v_mul_lo_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: mulmul_v2i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: mulmul_v2i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v1, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%gepa = getelementptr <2 x i16>, ptr addrspace(1) %ina, i32 %idx
@@ -484,16 +1776,79 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}add_bb_v2i16:
-; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
-
-; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-NEXT: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-NEXT: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-
-; GFX9_10: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
define amdgpu_kernel void @add_bb_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %ina, ptr addrspace(1) %inb) #0 {
+; NOSDWA-LABEL: add_bb_v2i16:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; NOSDWA-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s6
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s7
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s1
+; NOSDWA-NEXT: flat_load_dword v1, v[0:1]
+; NOSDWA-NEXT: flat_load_dword v2, v[2:3]
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT: s_waitcnt vmcnt(1)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; NOSDWA-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; NOSDWA-NEXT: v_add_u32_e32 v2, vcc, v3, v4
+; NOSDWA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; NOSDWA-NEXT: v_or_b32_e32 v2, v1, v2
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: flat_store_dword v[0:1], v2
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: add_bb_v2i16:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s6
+; GFX89-NEXT: v_mov_b32_e32 v1, s7
+; GFX89-NEXT: v_mov_b32_e32 v2, s0
+; GFX89-NEXT: v_mov_b32_e32 v3, s1
+; GFX89-NEXT: flat_load_dword v1, v[0:1]
+; GFX89-NEXT: flat_load_dword v2, v[2:3]
+; GFX89-NEXT: v_mov_b32_e32 v0, s4
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_add_u32_sdwa v3, vcc, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX89-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; GFX89-NEXT: v_or_b32_sdwa v2, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: v_mov_b32_e32 v1, s5
+; GFX89-NEXT: flat_store_dword v[0:1], v2
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_bb_v2i16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_pk_add_u16 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: add_bb_v2i16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_pk_add_u16 v1, v1, v2
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_endpgm
entry:
%a = load <2 x i16>, ptr addrspace(1) %ina, align 4
%b = load <2 x i16>, ptr addrspace(1) %inb, align 4
@@ -506,39 +1861,125 @@ store_label:
ret void
}
-
; Check that "pulling out" SDWA operands works correctly.
-; GCN-LABEL: {{^}}pulled_out_test:
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff, v{{[0-9]+}}
-; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff, v{{[0-9]+}}
-; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_and_b32_sdwa
-; NOSDWA-NOT: v_or_b32_sdwa
-
-; VI-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX89-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-;
-; GFX10-DAG: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-;
-; VI-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX89-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
+define amdgpu_kernel void @pulled_out_test(ptr addrspace(1) %sourceA, ptr addrspace(1) %destValues) #0 {
+; NOSDWA-LABEL: pulled_out_test:
+; NOSDWA: ; %bb.0: ; %entry
+; NOSDWA-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; NOSDWA-NEXT: s_waitcnt lgkmcnt(0)
+; NOSDWA-NEXT: v_mov_b32_e32 v0, s0
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s1
+; NOSDWA-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s2
+; NOSDWA-NEXT: v_mov_b32_e32 v3, s3
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: v_lshrrev_b64 v[4:5], 24, v[0:1]
+; NOSDWA-NEXT: v_and_b32_e32 v6, 0xff, v0
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v7, 8, v0
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_and_b32_e32 v5, 0xff, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; NOSDWA-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v7, 8, v7
+; NOSDWA-NEXT: v_and_b32_e32 v0, 0xff, v0
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v8, 8, v8
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v9, 8, v9
+; NOSDWA-NEXT: v_and_b32_e32 v1, 0xff, v1
+; NOSDWA-NEXT: v_lshlrev_b16_e32 v4, 8, v4
+; NOSDWA-NEXT: v_or_b32_e32 v6, v6, v7
+; NOSDWA-NEXT: v_or_b32_e32 v5, v5, v8
+; NOSDWA-NEXT: v_or_b32_e32 v1, v1, v9
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v4
+; NOSDWA-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; NOSDWA-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; NOSDWA-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; NOSDWA-NEXT: v_or_b32_e32 v0, v6, v0
+; NOSDWA-NEXT: v_or_b32_e32 v1, v4, v1
+; NOSDWA-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; NOSDWA-NEXT: s_endpgm
;
-; GFX10-DAG: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-LABEL: pulled_out_test:
+; GFX89: ; %bb.0: ; %entry
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT: v_mov_b32_e32 v6, 8
+; GFX89-NEXT: v_mov_b32_e32 v7, 0xff
+; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: v_mov_b32_e32 v0, s0
+; GFX89-NEXT: v_mov_b32_e32 v1, s1
+; GFX89-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX89-NEXT: v_mov_b32_e32 v2, s2
+; GFX89-NEXT: v_mov_b32_e32 v3, s3
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: v_lshrrev_b64 v[4:5], 24, v[0:1]
+; GFX89-NEXT: v_lshrrev_b32_sdwa v8, v6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-NEXT: v_lshrrev_b32_sdwa v6, v6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX89-NEXT: v_and_b32_sdwa v5, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-NEXT: v_and_b32_sdwa v7, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_lshlrev_b16_e32 v6, 8, v9
+; GFX89-NEXT: v_lshlrev_b16_e32 v4, 8, v4
+; GFX89-NEXT: v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX89-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX89-NEXT: s_endpgm
;
-; GFX89: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-LABEL: pulled_out_test:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX9-NEXT: s_movk_i32 s0, 0xff
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_sdwa v6, v5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT: v_and_b32_sdwa v3, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v7
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v5, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[2:3]
+; GFX9-NEXT: s_endpgm
;
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-
-define amdgpu_kernel void @pulled_out_test(ptr addrspace(1) %sourceA, ptr addrspace(1) %destValues) #0 {
+; GFX10-LABEL: pulled_out_test:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-NEXT: v_mov_b32_e32 v6, 0xff
+; GFX10-NEXT: v_mov_b32_e32 v7, 24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_sdwa v8, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v7, v7, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_sdwa v6, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
entry:
%idxprom = ashr exact i64 15, 32
%arrayidx = getelementptr inbounds <8 x i8>, ptr addrspace(1) %sourceA, i64 %idxprom
@@ -571,15 +2012,74 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}sdwa_crash_inlineasm_def:
-; GCN: s_mov_b32 s{{[0-9]+}}, 0xffff
-; GCN: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
-;
; TODO: Why is the constant not peepholed into the v_or_b32_e32?
-;
-; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, 0x10000,
-; SDWA: v_or_b32_e32 v{{[0-9]+}}, 0x10000,
define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
+; NOSDWA-LABEL: sdwa_crash_inlineasm_def:
+; NOSDWA: ; %bb.0: ; %bb
+; NOSDWA-NEXT: s_mov_b32 s0, 0xffff
+; NOSDWA-NEXT: ;;#ASMSTART
+; NOSDWA-NEXT: v_and_b32_e32 v0, s0, v0
+; NOSDWA-NEXT: ;;#ASMEND
+; NOSDWA-NEXT: v_or_b32_e32 v0, 0x10000, v0
+; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
+; NOSDWA-NEXT: .LBB21_1: ; %bb1
+; NOSDWA-NEXT: ; =>This Inner Loop Header: Depth=1
+; NOSDWA-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
+; NOSDWA-NEXT: s_waitcnt vmcnt(0)
+; NOSDWA-NEXT: s_mov_b64 vcc, vcc
+; NOSDWA-NEXT: s_cbranch_vccnz .LBB21_1
+; NOSDWA-NEXT: ; %bb.2: ; %DummyReturnBlock
+; NOSDWA-NEXT: s_endpgm
+;
+; GFX89-LABEL: sdwa_crash_inlineasm_def:
+; GFX89: ; %bb.0: ; %bb
+; GFX89-NEXT: s_mov_b32 s0, 0xffff
+; GFX89-NEXT: ;;#ASMSTART
+; GFX89-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX89-NEXT: ;;#ASMEND
+; GFX89-NEXT: v_or_b32_e32 v0, 0x10000, v0
+; GFX89-NEXT: s_and_b64 vcc, exec, -1
+; GFX89-NEXT: .LBB21_1: ; %bb1
+; GFX89-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX89-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
+; GFX89-NEXT: s_waitcnt vmcnt(0)
+; GFX89-NEXT: s_mov_b64 vcc, vcc
+; GFX89-NEXT: s_cbranch_vccnz .LBB21_1
+; GFX89-NEXT: ; %bb.2: ; %DummyReturnBlock
+; GFX89-NEXT: s_endpgm
+;
+; GFX9-LABEL: sdwa_crash_inlineasm_def:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_mov_b32 s0, 0xffff
+; GFX9-NEXT: ;;#ASMSTART
+; GFX9-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX9-NEXT: ;;#ASMEND
+; GFX9-NEXT: v_or_b32_e32 v0, 0x10000, v0
+; GFX9-NEXT: s_and_b64 vcc, exec, -1
+; GFX9-NEXT: .LBB21_1: ; %bb1
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: global_store_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_mov_b64 vcc, vcc
+; GFX9-NEXT: s_cbranch_vccnz .LBB21_1
+; GFX9-NEXT: ; %bb.2: ; %DummyReturnBlock
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: sdwa_crash_inlineasm_def:
+; GFX10: ; %bb.0: ; %bb
+; GFX10-NEXT: s_mov_b32 s0, 0xffff
+; GFX10-NEXT: ;;#ASMSTART
+; GFX10-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX10-NEXT: ;;#ASMEND
+; GFX10-NEXT: v_or_b32_e32 v0, 0x10000, v0
+; GFX10-NEXT: s_mov_b32 vcc_lo, exec_lo
+; GFX10-NEXT: .LBB21_1: ; %bb1
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_cbranch_vccnz .LBB21_1
+; GFX10-NEXT: ; %bb.2: ; %DummyReturnBlock
+; GFX10-NEXT: s_endpgm
bb:
br label %bb1
@@ -602,3 +2102,7 @@ bb11: ; preds = %bb10, %bb2
declare i32 @llvm.amdgcn.workitem.id.x()
attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
+; GFX9_10: {{.*}}
+; SDWA: {{.*}}
More information about the llvm-commits
mailing list