[llvm] [NFC][SLP] Precommit tests for the phantom load saving and the cost context (PR #228901)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 08:12:40 PDT 2026
https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/228901
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From dd03154318e101a12e1c9146f51926effab2d464 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 3 Oct 2026 14:20:25 +0200
Subject: [PATCH] [NFC][SLP] Precommit tests for the phantom load saving and
the cost context
---
.../CodeGen/AMDGPU/slp-coalesced-loads.ll | 1536 +++++++++++++++++
.../AMDGPU/elementwise-coalesced-loads.ll | 454 +++++
.../AMDGPU/fdiv-afn-reduction.ll | 70 +
.../AMDGPU/int-reduction-coalesced-loads.ll | 527 ++++++
.../ordered-reduction-coalesced-loads.ll | 684 ++++++++
.../reassoc-reduction-coalesced-loads.ll | 159 ++
.../SLPVectorizer/X86/udiv-strictfp.ll | 112 ++
7 files changed, 3542 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-reduction.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/reassoc-reduction-coalesced-loads.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/udiv-strictfp.ll
diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
new file mode 100644
index 0000000000000..4ac9d6adf090e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
@@ -0,0 +1,1536 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgpu9.0a-amd-amdhsa -passes=slp-vectorizer < %s | llc -mtriple=amdgpu9.0a-amd-amdhsa | FileCheck -check-prefix=GFX90A %s
+; RUN: opt -mtriple=amdgpu10.30-amd-amdhsa -passes=slp-vectorizer < %s | llc -mtriple=amdgpu10.30-amd-amdhsa | FileCheck -check-prefix=GFX1030 %s
+; RUN: opt -mtriple=amdgpu12.50-amd-amdhsa -passes=slp-vectorizer < %s | llc -mtriple=amdgpu12.50-amd-amdhsa | FileCheck -check-prefix=GFX1250 %s
+
+define float @dot8_ordered(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b, float %acc) {
+; GFX90A-LABEL: dot8_ordered:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[14:17], v[2:3], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[18:21], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[18:19]
+; GFX90A-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[8:9], v[20:21]
+; GFX90A-NEXT: v_add_f32_e32 v4, v4, v7
+; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
+; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[10:11], v[14:15]
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v8
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_ordered:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx4 v[5:8], v[2:3], off
+; GFX1030-NEXT: s_clause 0x1
+; GFX1030-NEXT: global_load_dwordx4 v[9:12], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx4 v[13:16], v[0:1], off offset:16
+; GFX1030-NEXT: global_load_dwordx4 v[17:20], v[2:3], off offset:16
+; GFX1030-NEXT: s_waitcnt vmcnt(2)
+; GFX1030-NEXT: v_fma_f32 v0, v9, v5, v4
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v11, v7
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v12, v8
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v15, v19
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v16, v20
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_ordered:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_b128 v[10:13], v[0:1], off
+; GFX1250-NEXT: global_load_b128 v[14:17], v[0:1], off offset:16
+; GFX1250-NEXT: global_load_b128 v[18:21], v[2:3], off offset:16
+; GFX1250-NEXT: s_wait_loadcnt 0x2
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[6:7]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[12:13], v[8:9]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[14:15], v[18:19]
+; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[16:17], v[20:21]
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s0 = fadd contract float %acc, %m0
+ %s1 = fadd contract float %s0, %m1
+ %s2 = fadd contract float %s1, %m2
+ %s3 = fadd contract float %s2, %m3
+ %s4 = fadd contract float %s3, %m4
+ %s5 = fadd contract float %s4, %m5
+ %s6 = fadd contract float %s5, %m6
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+
+define float @dot8_reassoc(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b) {
+; GFX90A-LABEL: dot8_reassoc:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[12:15], v[2:3], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[16:19], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[14:15]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_pk_fma_f32 v[0:1], v[6:7], v[18:19], v[0:1]
+; GFX90A-NEXT: v_pk_fma_f32 v[0:1], v[8:9], v[12:13], v[0:1]
+; GFX90A-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[16:17], v[0:1]
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_reassoc:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; GFX1030-NEXT: s_clause 0x1
+; GFX1030-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX1030-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx4 v[16:19], v[2:3], off
+; GFX1030-NEXT: s_waitcnt vmcnt(2)
+; GFX1030-NEXT: v_mul_f32_e32 v0, v11, v7
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v15, v19
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v9, v5
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_reassoc:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[4:7], v[2:3], off offset:16
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
+; GFX1250-NEXT: global_load_b128 v[12:15], v[0:1], off
+; GFX1250-NEXT: global_load_b128 v[16:19], v[2:3], off
+; GFX1250-NEXT: s_wait_loadcnt 0x2
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[6:7]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[8:9], v[4:5]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[14:15], v[18:19], v[0:1]
+; GFX1250-NEXT: v_pk_fma_f32 v[2:3], v[12:13], v[16:17], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s1 = fadd fast float %m0, %m1
+ %s2 = fadd fast float %s1, %m2
+ %s3 = fadd fast float %s2, %m3
+ %s4 = fadd fast float %s3, %m4
+ %s5 = fadd fast float %s4, %m5
+ %s6 = fadd fast float %s5, %m6
+ %s7 = fadd fast float %s6, %m7
+ ret float %s7
+}
+
+define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
+; GFX90A-LABEL: dot_i16_window:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx2 v[10:11], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
+; GFX90A-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[8:9]
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot_i16_window:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx2 v[5:6], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx4 v[0:3], v[2:3], off
+; GFX1030-NEXT: s_waitcnt vmcnt(1)
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v7, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_fma_f32 v0, v7, v0, v4
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v5, v1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v1, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v4, v2
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v1, v3
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot_i16_window:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[10:11], v[0:1], off
+; GFX1250-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX1250-NEXT: s_wait_loadcnt 0x1
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_ashrrev_i32_e32 v0, 16, v10
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v2, v10, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v5, v11, 0, 16
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v2
+; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 16, v11
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v2
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[8:9]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
+ %a2 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 4
+ %a3 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 6
+ %b1 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 4
+ %b2 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 8
+ %b3 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 12
+ %i0 = load i16, ptr addrspace(1) %a, align 2
+ %i1 = load i16, ptr addrspace(1) %a1, align 2
+ %i2 = load i16, ptr addrspace(1) %a2, align 2
+ %i3 = load i16, ptr addrspace(1) %a3, align 2
+ %f0 = sitofp i16 %i0 to float
+ %f1 = sitofp i16 %i1 to float
+ %f2 = sitofp i16 %i2 to float
+ %f3 = sitofp i16 %i3 to float
+ %w0 = load float, ptr addrspace(1) %b, align 4
+ %w1 = load float, ptr addrspace(1) %b1, align 4
+ %w2 = load float, ptr addrspace(1) %b2, align 4
+ %w3 = load float, ptr addrspace(1) %b3, align 4
+ %m0 = fmul contract float %f0, %w0
+ %m1 = fmul contract float %f1, %w1
+ %m2 = fmul contract float %f2, %w2
+ %m3 = fmul contract float %f3, %w3
+ %s0 = fadd contract float %acc, %m0
+ %s1 = fadd contract float %s0, %m1
+ %s2 = fadd contract float %s1, %m2
+ %s3 = fadd contract float %s2, %m3
+ ret float %s3
+}
+
+define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX90A-LABEL: store4_sitofp:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v0, v[0:1], off
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_ashrrev_i16_e32 v1, 8, v0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: store4_sitofp:
+; GFX1030: ; %bb.0:
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dword v0, v[0:1], off
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX1030-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: store4_sitofp:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v5, v0, 8, 8
+; GFX1250-NEXT: v_bfe_i32 v6, v0, 16, 8
+; GFX1250-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v1
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v6
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v0
+; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
+ %q0 = load i8, ptr addrspace(1) %in, align 4
+ %q1 = load i8, ptr addrspace(1) %p1, align 1
+ %q2 = load i8, ptr addrspace(1) %p2, align 2
+ %q3 = load i8, ptr addrspace(1) %p3, align 1
+ %f0 = sitofp i8 %q0 to float
+ %f1 = sitofp i8 %q1 to float
+ %f2 = sitofp i8 %q2 to float
+ %f3 = sitofp i8 %q3 to float
+ %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
+ store float %f0, ptr addrspace(1) %out, align 16
+ store float %f1, ptr addrspace(1) %o1, align 4
+ store float %f2, ptr addrspace(1) %o2, align 8
+ store float %f3, ptr addrspace(1) %o3, align 4
+ ret void
+}
+
+define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; GFX90A-LABEL: zext4_bv_loop:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: .LBB4_1: ; %loop
+; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: global_load_dword v6, v[0:1], off
+; GFX90A-NEXT: s_add_u32 s6, s6, 1
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-NEXT: s_addc_u32 s7, s7, 0
+; GFX90A-NEXT: v_cmp_lt_u64_e64 vcc, s[6:7], 16
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b32_e32 v7, 24, v6
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xff, v6
+; GFX90A-NEXT: v_lshrrev_b16_e32 v5, 8, v6
+; GFX90A-NEXT: v_bfe_u32 v6, v6, 16, 8
+; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX90A-NEXT: v_add_co_u32_e64 v2, s[4:5], 16, v2
+; GFX90A-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, v3, s[4:5]
+; GFX90A-NEXT: s_cbranch_vccnz .LBB4_1
+; GFX90A-NEXT: ; %bb.2: ; %exit
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: zext4_bv_loop:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: v_mov_b32_e32 v4, 0xffff
+; GFX1030-NEXT: s_mov_b64 s[6:7], 0
+; GFX1030-NEXT: .p2align 6
+; GFX1030-NEXT: .LBB4_1: ; %loop
+; GFX1030-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1030-NEXT: global_load_dword v7, v[0:1], off
+; GFX1030-NEXT: s_add_u32 s6, s6, 1
+; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 4
+; GFX1030-NEXT: s_addc_u32 s7, s7, 0
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-NEXT: v_cmp_lt_u64_e64 vcc_lo, s[6:7], 16
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b32_e32 v8, 24, v7
+; GFX1030-NEXT: v_and_b32_e32 v5, 0xff, v7
+; GFX1030-NEXT: v_and_b32_sdwa v6, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX1030-NEXT: v_bfe_u32 v7, v7, 16, 8
+; GFX1030-NEXT: global_store_dwordx4 v[2:3], v[5:8], off
+; GFX1030-NEXT: v_add_co_u32 v2, s4, v2, 16
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s4
+; GFX1030-NEXT: s_cbranch_vccnz .LBB4_1
+; GFX1030-NEXT: ; %bb.2: ; %exit
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: zext4_bv_loop:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0
+; GFX1250-NEXT: .LBB4_1: ; %loop
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 1
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 4, v[0:1]
+; GFX1250-NEXT: v_cmp_lt_u64_e64 vcc_lo, s[0:1], 16
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b16 v5.l, 8, v6.l
+; GFX1250-NEXT: s_mov_b32 s2, 0xff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_dual_lshrrev_b32 v7, 24, v6 :: v_dual_bitop2_b32 v4, s2, v6 bitop3:0x40
+; GFX1250-NEXT: v_bfe_u32 v6, v6, 16, 8
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v5, v5.l
+; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 16, v[2:3]
+; GFX1250-NEXT: s_cbranch_vccnz .LBB4_1
+; GFX1250-NEXT: ; %bb.2: ; %exit
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %off = shl i64 %i, 2
+ %p0 = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %off
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 3
+ %l0 = load i8, ptr addrspace(1) %p0, align 4
+ %l1 = load i8, ptr addrspace(1) %p1, align 1
+ %l2 = load i8, ptr addrspace(1) %p2, align 2
+ %l3 = load i8, ptr addrspace(1) %p3, align 1
+ %x0 = zext i8 %l0 to i32
+ %x1 = zext i8 %l1 to i32
+ %x2 = zext i8 %l2 to i32
+ %x3 = zext i8 %l3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ %po = getelementptr inbounds <4 x i32>, ptr addrspace(1) %out, i64 %i
+ store <4 x i32> %v3, ptr addrspace(1) %po, align 16
+ %i.next = add nuw nsw i64 %i, 1
+ %c = icmp ult i64 %i.next, 16
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
+
+define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX90A-LABEL: dot8_sext_i8:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_lshrrev_b16_e32 v0, 8, v4
+; GFX90A-NEXT: v_lshrrev_b16_e32 v1, 8, v5
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v6
+; GFX90A-NEXT: v_lshrrev_b16_e32 v3, 8, v7
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v9, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v10, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_add_u32_e32 v4, v4, v5
+; GFX90A-NEXT: v_add_u32_e32 v2, v9, v10
+; GFX90A-NEXT: v_add_u32_e32 v3, v8, v6
+; GFX90A-NEXT: v_add3_u32 v0, v0, v1, v4
+; GFX90A-NEXT: v_add3_u32 v0, v3, v2, v0
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_sext_i8:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX1030-NEXT: s_waitcnt vmcnt(1)
+; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v0
+; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v1
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v2
+; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v3
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1
+; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0
+; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_sext_i8:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX1250-NEXT: global_load_b64 v[6:7], v[2:3], off
+; GFX1250-NEXT: s_wait_loadcnt 0x1
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v4.l
+; GFX1250-NEXT: v_lshrrev_b16 v1.l, 8, v5.l
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v6.l
+; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v7.l
+; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v5 :: v_dual_ashrrev_i32 v14, 24, v7
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_ashrrev_i32 v2, 24, v4 :: v_dual_ashrrev_i32 v10, 24, v6
+; GFX1250-NEXT: v_bfe_i32 v13, v5, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v15, v7, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14
+; GFX1250-NEXT: v_bfe_i32 v3, v4, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v4, v4, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v11, v6, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v6, v6, 16, 8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v7
+; GFX1250-NEXT: v_mul_i32_i24_e32 v7, v13, v15
+; GFX1250-NEXT: v_mul_i32_i24_e32 v0, v0, v8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v1, v9
+; GFX1250-NEXT: v_mad_i32_i24 v2, v2, v10, v12
+; GFX1250-NEXT: v_mad_i32_i24 v3, v3, v11, v5
+; GFX1250-NEXT: v_mad_i32_i24 v4, v4, v6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX1250-NEXT: v_add3_u32 v0, v3, v4, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = sext i8 %la0 to i32
+ %pb0 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 0
+ %lb0 = load i8, ptr addrspace(1) %pb0, align 1
+ %xb0 = sext i8 %lb0 to i32
+ %m0 = mul nsw i32 %xa0, %xb0
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = sext i8 %la1 to i32
+ %pb1 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 1
+ %lb1 = load i8, ptr addrspace(1) %pb1, align 1
+ %xb1 = sext i8 %lb1 to i32
+ %m1 = mul nsw i32 %xa1, %xb1
+ %r1 = add nsw i32 %m0, %m1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = sext i8 %la2 to i32
+ %pb2 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 2
+ %lb2 = load i8, ptr addrspace(1) %pb2, align 1
+ %xb2 = sext i8 %lb2 to i32
+ %m2 = mul nsw i32 %xa2, %xb2
+ %r2 = add nsw i32 %r1, %m2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = sext i8 %la3 to i32
+ %pb3 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 3
+ %lb3 = load i8, ptr addrspace(1) %pb3, align 1
+ %xb3 = sext i8 %lb3 to i32
+ %m3 = mul nsw i32 %xa3, %xb3
+ %r3 = add nsw i32 %r2, %m3
+ %pa4 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 4
+ %la4 = load i8, ptr addrspace(1) %pa4, align 1
+ %xa4 = sext i8 %la4 to i32
+ %pb4 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 4
+ %lb4 = load i8, ptr addrspace(1) %pb4, align 1
+ %xb4 = sext i8 %lb4 to i32
+ %m4 = mul nsw i32 %xa4, %xb4
+ %r4 = add nsw i32 %r3, %m4
+ %pa5 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 5
+ %la5 = load i8, ptr addrspace(1) %pa5, align 1
+ %xa5 = sext i8 %la5 to i32
+ %pb5 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 5
+ %lb5 = load i8, ptr addrspace(1) %pb5, align 1
+ %xb5 = sext i8 %lb5 to i32
+ %m5 = mul nsw i32 %xa5, %xb5
+ %r5 = add nsw i32 %r4, %m5
+ %pa6 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 6
+ %la6 = load i8, ptr addrspace(1) %pa6, align 1
+ %xa6 = sext i8 %la6 to i32
+ %pb6 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 6
+ %lb6 = load i8, ptr addrspace(1) %pb6, align 1
+ %xb6 = sext i8 %lb6 to i32
+ %m6 = mul nsw i32 %xa6, %xb6
+ %r6 = add nsw i32 %r5, %m6
+ %pa7 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 7
+ %la7 = load i8, ptr addrspace(1) %pa7, align 1
+ %xa7 = sext i8 %la7 to i32
+ %pb7 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 7
+ %lb7 = load i8, ptr addrspace(1) %pb7, align 1
+ %xb7 = sext i8 %lb7 to i32
+ %m7 = mul nsw i32 %xa7, %xb7
+ %r7 = add nsw i32 %r6, %m7
+ ret i32 %r7
+}
+
+define i32 @sum4_zext_i8(ptr addrspace(1) %a) {
+; GFX90A-LABEL: sum4_zext_i8:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v0, v[0:1], off
+; GFX90A-NEXT: s_movk_i32 s4, 0xff
+; GFX90A-NEXT: s_mov_b32 s5, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 24, v0
+; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v0
+; GFX90A-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX90A-NEXT: v_perm_b32 v0, v2, v0, s5
+; GFX90A-NEXT: v_perm_b32 v1, v1, v3, s5
+; GFX90A-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX90A-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: sum4_zext_i8:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dword v0, v[0:1], off
+; GFX1030-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b32_e32 v2, 24, v0
+; GFX1030-NEXT: v_lshrrev_b16 v3, 8, v0
+; GFX1030-NEXT: v_and_b32_e32 v4, 0xff, v0
+; GFX1030-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX1030-NEXT: v_perm_b32 v1, v3, v4, 0x5040100
+; GFX1030-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
+; GFX1030-NEXT: v_pk_add_u16 v0, v1, v0
+; GFX1030-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1030-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX1030-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: sum4_zext_i8:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v1, 24, v0
+; GFX1250-NEXT: v_lshrrev_b16 v2.h, 8, v0.l
+; GFX1250-NEXT: v_and_b16 v3.l, 0xff, v0.h
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v0.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX1250-NEXT: v_pk_add_u16 v0, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = zext i8 %la0 to i32
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = zext i8 %la1 to i32
+ %r1 = add nsw i32 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = zext i8 %la2 to i32
+ %r2 = add nsw i32 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = zext i8 %la3 to i32
+ %r3 = add nsw i32 %r2, %xa3
+ ret i32 %r3
+}
+
+define i16 @sum4_sext_i8_i16(ptr addrspace(1) %a) {
+; GFX90A-LABEL: sum4_sext_i8_i16:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dword v0, v[0:1], off
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX90A-NEXT: v_ashrrev_i32_e32 v2, 24, v0
+; GFX90A-NEXT: v_ashrrev_i16_e32 v3, 8, v0
+; GFX90A-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX90A-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s4
+; GFX90A-NEXT: v_perm_b32 v1, v2, v1, s4
+; GFX90A-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX90A-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: sum4_sext_i8_i16:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dword v0, v[0:1], off
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1030-NEXT: v_ashrrev_i32_e32 v2, 24, v0
+; GFX1030-NEXT: v_ashrrev_i16 v3, 8, v0
+; GFX1030-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX1030-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX1030-NEXT: v_perm_b32 v0, v3, v0, 0x5040100
+; GFX1030-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX1030-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX1030-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1030-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: sum4_sext_i8_i16:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_bfe_i32 v2, v0, 0, 8
+; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 24, v0
+; GFX1250-NEXT: v_ashrrev_i16 v0.h, 8, v0.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX1250-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = sext i8 %la0 to i16
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = sext i8 %la1 to i16
+ %r1 = add nsw i16 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = sext i8 %la2 to i16
+ %r2 = add nsw i16 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = sext i8 %la3 to i16
+ %r3 = add nsw i16 %r2, %xa3
+ ret i16 %r3
+}
+
+define void @store8_sitofp_half(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX90A-LABEL: store8_sitofp_half:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX90A-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX90A-NEXT: v_ashrrev_i16_e32 v4, 8, v0
+; GFX90A-NEXT: v_ashrrev_i16_e32 v6, 8, v1
+; GFX90A-NEXT: v_ashrrev_i16_e32 v8, 8, v5
+; GFX90A-NEXT: v_ashrrev_i16_e32 v9, 8, v7
+; GFX90A-NEXT: v_cvt_f16_i16_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX90A-NEXT: v_cvt_f16_i16_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX90A-NEXT: v_cvt_f16_i16_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX90A-NEXT: v_cvt_f16_i16_e32 v4, v4
+; GFX90A-NEXT: v_cvt_f16_i16_sdwa v7, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX90A-NEXT: v_cvt_f16_i16_e32 v6, v6
+; GFX90A-NEXT: v_cvt_f16_i16_e32 v8, v8
+; GFX90A-NEXT: v_cvt_f16_i16_e32 v9, v9
+; GFX90A-NEXT: v_pack_b32_f16 v6, v1, v6
+; GFX90A-NEXT: v_pack_b32_f16 v4, v0, v4
+; GFX90A-NEXT: v_pack_b32_f16 v7, v7, v9
+; GFX90A-NEXT: v_pack_b32_f16 v5, v5, v8
+; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: store8_sitofp_half:
+; GFX1030: ; %bb.0:
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX1030-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX1030-NEXT: v_ashrrev_i16 v4, 8, v0
+; GFX1030-NEXT: v_ashrrev_i16 v7, 8, v1
+; GFX1030-NEXT: v_cvt_f16_i16_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX1030-NEXT: v_ashrrev_i16 v8, 8, v5
+; GFX1030-NEXT: v_ashrrev_i16 v9, 8, v6
+; GFX1030-NEXT: v_cvt_f16_i16_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX1030-NEXT: v_cvt_f16_i16_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX1030-NEXT: v_cvt_f16_i16_e32 v4, v4
+; GFX1030-NEXT: v_cvt_f16_i16_e32 v7, v7
+; GFX1030-NEXT: v_cvt_f16_i16_sdwa v10, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX1030-NEXT: v_cvt_f16_i16_e32 v9, v9
+; GFX1030-NEXT: v_cvt_f16_i16_e32 v8, v8
+; GFX1030-NEXT: v_pack_b32_f16 v4, v0, v4
+; GFX1030-NEXT: v_pack_b32_f16 v6, v1, v7
+; GFX1030-NEXT: v_pack_b32_f16 v7, v10, v9
+; GFX1030-NEXT: v_pack_b32_f16 v5, v5, v8
+; GFX1030-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: store8_sitofp_half:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX1250-NEXT: v_dual_ashrrev_i32 v6, 24, v4 :: v_dual_ashrrev_i32 v8, 24, v5
+; GFX1250-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX1250-NEXT: v_bfe_i32 v9, v5, 0, 8
+; GFX1250-NEXT: v_ashrrev_i16 v0.l, 8, v4.l
+; GFX1250-NEXT: v_bfe_i32 v10, v4, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v11, v1, 0, 8
+; GFX1250-NEXT: v_ashrrev_i16 v0.h, 8, v5.l
+; GFX1250-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v4.h, v0.l
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, v10.l
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v9.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v7.h, v8.l
+; GFX1250-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v6.h, v0.h
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v5.h, v6.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v4.l, v0.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v6.l, v1.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v7.l, v5.l
+; GFX1250-NEXT: v_cvt_f16_i16_e32 v5.l, v8.l
+; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 4
+ %p5 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 5
+ %p6 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 6
+ %p7 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 7
+ %q0 = load i8, ptr addrspace(1) %in, align 8
+ %q1 = load i8, ptr addrspace(1) %p1, align 1
+ %q2 = load i8, ptr addrspace(1) %p2, align 2
+ %q3 = load i8, ptr addrspace(1) %p3, align 1
+ %q4 = load i8, ptr addrspace(1) %p4, align 4
+ %q5 = load i8, ptr addrspace(1) %p5, align 1
+ %q6 = load i8, ptr addrspace(1) %p6, align 2
+ %q7 = load i8, ptr addrspace(1) %p7, align 1
+ %f0 = sitofp i8 %q0 to half
+ %f1 = sitofp i8 %q1 to half
+ %f2 = sitofp i8 %q2 to half
+ %f3 = sitofp i8 %q3 to half
+ %f4 = sitofp i8 %q4 to half
+ %f5 = sitofp i8 %q5 to half
+ %f6 = sitofp i8 %q6 to half
+ %f7 = sitofp i8 %q7 to half
+ %o1 = getelementptr inbounds half, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds half, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds half, ptr addrspace(1) %out, i64 3
+ %o4 = getelementptr inbounds half, ptr addrspace(1) %out, i64 4
+ %o5 = getelementptr inbounds half, ptr addrspace(1) %out, i64 5
+ %o6 = getelementptr inbounds half, ptr addrspace(1) %out, i64 6
+ %o7 = getelementptr inbounds half, ptr addrspace(1) %out, i64 7
+ store half %f0, ptr addrspace(1) %out, align 16
+ store half %f1, ptr addrspace(1) %o1, align 2
+ store half %f2, ptr addrspace(1) %o2, align 4
+ store half %f3, ptr addrspace(1) %o3, align 2
+ store half %f4, ptr addrspace(1) %o4, align 8
+ store half %f5, ptr addrspace(1) %o5, align 2
+ store half %f6, ptr addrspace(1) %o6, align 4
+ store half %f7, ptr addrspace(1) %o7, align 2
+ ret void
+}
+
+define i32 @dot8_sext_i8_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b) {
+; GFX90A-LABEL: dot8_sext_i8_lds_a16:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: ds_read_b64 v[2:3], v0
+; GFX90A-NEXT: ds_read_b64 v[0:1], v1
+; GFX90A-NEXT: s_waitcnt lgkmcnt(1)
+; GFX90A-NEXT: v_lshrrev_b16_e32 v4, 8, v2
+; GFX90A-NEXT: v_lshrrev_b16_e32 v5, 8, v3
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: v_lshrrev_b16_e32 v6, 8, v0
+; GFX90A-NEXT: v_lshrrev_b16_e32 v7, 8, v1
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v7, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX90A-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX90A-NEXT: v_add_u32_e32 v1, v6, v7
+; GFX90A-NEXT: v_add_u32_e32 v2, v8, v2
+; GFX90A-NEXT: v_add3_u32 v0, v4, v5, v0
+; GFX90A-NEXT: v_add3_u32 v0, v2, v1, v0
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_sext_i8_lds_a16:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: ds_read_b64 v[2:3], v0
+; GFX1030-NEXT: ds_read_b64 v[0:1], v1
+; GFX1030-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v2
+; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v3
+; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v0
+; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v1
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1
+; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0
+; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_sext_i8_lds_a16:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: ds_load_b64 v[2:3], v0
+; GFX1250-NEXT: ds_load_b64 v[0:1], v1
+; GFX1250-NEXT: s_wait_dscnt 0x1
+; GFX1250-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX1250-NEXT: v_lshrrev_b16 v5.l, 8, v3.l
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v0.l
+; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v1.l
+; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v3 :: v_dual_ashrrev_i32 v14, 24, v1
+; GFX1250-NEXT: v_dual_ashrrev_i32 v6, 24, v2 :: v_dual_ashrrev_i32 v10, 24, v0
+; GFX1250-NEXT: v_bfe_i32 v13, v3, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v15, v1, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14
+; GFX1250-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v2, v2, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX1250-NEXT: v_bfe_i32 v0, v0, 16, 8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v3, v1
+; GFX1250-NEXT: v_mul_i32_i24_e32 v3, v13, v15
+; GFX1250-NEXT: v_mul_i32_i24_e32 v4, v4, v8
+; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v9
+; GFX1250-NEXT: v_mad_i32_i24 v6, v6, v10, v12
+; GFX1250-NEXT: v_mad_i32_i24 v1, v7, v11, v1
+; GFX1250-NEXT: v_mad_i32_i24 v0, v2, v0, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add3_u32 v2, v4, v5, v6
+; GFX1250-NEXT: v_add3_u32 v0, v1, v0, v2
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 0
+ %la0 = load i8, ptr addrspace(3) %pa0, align 16
+ %xa0 = sext i8 %la0 to i32
+ %pb0 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 0
+ %lb0 = load i8, ptr addrspace(3) %pb0, align 16
+ %xb0 = sext i8 %lb0 to i32
+ %m0 = mul nsw i32 %xa0, %xb0
+ %pa1 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 1
+ %la1 = load i8, ptr addrspace(3) %pa1, align 1
+ %xa1 = sext i8 %la1 to i32
+ %pb1 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 1
+ %lb1 = load i8, ptr addrspace(3) %pb1, align 1
+ %xb1 = sext i8 %lb1 to i32
+ %m1 = mul nsw i32 %xa1, %xb1
+ %r1 = add nsw i32 %m0, %m1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 2
+ %la2 = load i8, ptr addrspace(3) %pa2, align 2
+ %xa2 = sext i8 %la2 to i32
+ %pb2 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 2
+ %lb2 = load i8, ptr addrspace(3) %pb2, align 2
+ %xb2 = sext i8 %lb2 to i32
+ %m2 = mul nsw i32 %xa2, %xb2
+ %r2 = add nsw i32 %r1, %m2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 3
+ %la3 = load i8, ptr addrspace(3) %pa3, align 1
+ %xa3 = sext i8 %la3 to i32
+ %pb3 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 3
+ %lb3 = load i8, ptr addrspace(3) %pb3, align 1
+ %xb3 = sext i8 %lb3 to i32
+ %m3 = mul nsw i32 %xa3, %xb3
+ %r3 = add nsw i32 %r2, %m3
+ %pa4 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 4
+ %la4 = load i8, ptr addrspace(3) %pa4, align 4
+ %xa4 = sext i8 %la4 to i32
+ %pb4 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 4
+ %lb4 = load i8, ptr addrspace(3) %pb4, align 4
+ %xb4 = sext i8 %lb4 to i32
+ %m4 = mul nsw i32 %xa4, %xb4
+ %r4 = add nsw i32 %r3, %m4
+ %pa5 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 5
+ %la5 = load i8, ptr addrspace(3) %pa5, align 1
+ %xa5 = sext i8 %la5 to i32
+ %pb5 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 5
+ %lb5 = load i8, ptr addrspace(3) %pb5, align 1
+ %xb5 = sext i8 %lb5 to i32
+ %m5 = mul nsw i32 %xa5, %xb5
+ %r5 = add nsw i32 %r4, %m5
+ %pa6 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 6
+ %la6 = load i8, ptr addrspace(3) %pa6, align 2
+ %xa6 = sext i8 %la6 to i32
+ %pb6 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 6
+ %lb6 = load i8, ptr addrspace(3) %pb6, align 2
+ %xb6 = sext i8 %lb6 to i32
+ %m6 = mul nsw i32 %xa6, %xb6
+ %r6 = add nsw i32 %r5, %m6
+ %pa7 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 7
+ %la7 = load i8, ptr addrspace(3) %pa7, align 1
+ %xa7 = sext i8 %la7 to i32
+ %pb7 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 7
+ %lb7 = load i8, ptr addrspace(3) %pb7, align 1
+ %xb7 = sext i8 %lb7 to i32
+ %m7 = mul nsw i32 %xa7, %xb7
+ %r7 = add nsw i32 %r6, %m7
+ ret i32 %r7
+}
+
+define float @dot8_ordered_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b, float %acc) {
+; GFX90A-LABEL: dot8_ordered_lds_a16:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: ds_read_b128 v[4:7], v0
+; GFX90A-NEXT: ds_read_b128 v[8:11], v0 offset:16
+; GFX90A-NEXT: ds_read_b128 v[12:15], v1 offset:16
+; GFX90A-NEXT: ds_read_b128 v[16:19], v1
+; GFX90A-NEXT: s_waitcnt lgkmcnt(1)
+; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[12:13]
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[18:19]
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v7
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v8
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[14:15]
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_ordered_lds_a16:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: ds_read_b128 v[3:6], v1
+; GFX1030-NEXT: ds_read_b128 v[7:10], v0
+; GFX1030-NEXT: ds_read_b128 v[11:14], v0 offset:16
+; GFX1030-NEXT: ds_read_b128 v[15:18], v1 offset:16
+; GFX1030-NEXT: s_waitcnt lgkmcnt(2)
+; GFX1030-NEXT: v_fma_f32 v0, v7, v3, v2
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v9, v5
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v11, v15
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_ordered_lds_a16:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: ds_load_b128 v[4:7], v1
+; GFX1250-NEXT: ds_load_b128 v[8:11], v0
+; GFX1250-NEXT: ds_load_b128 v[12:15], v0 offset:16
+; GFX1250-NEXT: ds_load_b128 v[16:19], v1 offset:16
+; GFX1250-NEXT: s_wait_dscnt 0x2
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[8:9], v[4:5]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[10:11], v[6:7]
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
+; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[14:15], v[18:19]
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(3) %a, i32 0
+ %la0 = load float, ptr addrspace(3) %pa0, align 16
+ %pb0 = getelementptr inbounds float, ptr addrspace(3) %b, i32 0
+ %lb0 = load float, ptr addrspace(3) %pb0, align 16
+ %m0 = fmul contract float %la0, %lb0
+ %s0 = fadd contract float %acc, %m0
+ %pa1 = getelementptr inbounds float, ptr addrspace(3) %a, i32 1
+ %la1 = load float, ptr addrspace(3) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(3) %b, i32 1
+ %lb1 = load float, ptr addrspace(3) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %s1 = fadd contract float %s0, %m1
+ %pa2 = getelementptr inbounds float, ptr addrspace(3) %a, i32 2
+ %la2 = load float, ptr addrspace(3) %pa2, align 8
+ %pb2 = getelementptr inbounds float, ptr addrspace(3) %b, i32 2
+ %lb2 = load float, ptr addrspace(3) %pb2, align 8
+ %m2 = fmul contract float %la2, %lb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds float, ptr addrspace(3) %a, i32 3
+ %la3 = load float, ptr addrspace(3) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(3) %b, i32 3
+ %lb3 = load float, ptr addrspace(3) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds float, ptr addrspace(3) %a, i32 4
+ %la4 = load float, ptr addrspace(3) %pa4, align 16
+ %pb4 = getelementptr inbounds float, ptr addrspace(3) %b, i32 4
+ %lb4 = load float, ptr addrspace(3) %pb4, align 16
+ %m4 = fmul contract float %la4, %lb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds float, ptr addrspace(3) %a, i32 5
+ %la5 = load float, ptr addrspace(3) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(3) %b, i32 5
+ %lb5 = load float, ptr addrspace(3) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds float, ptr addrspace(3) %a, i32 6
+ %la6 = load float, ptr addrspace(3) %pa6, align 8
+ %pb6 = getelementptr inbounds float, ptr addrspace(3) %b, i32 6
+ %lb6 = load float, ptr addrspace(3) %pb6, align 8
+ %m6 = fmul contract float %la6, %lb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds float, ptr addrspace(3) %a, i32 7
+ %la7 = load float, ptr addrspace(3) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(3) %b, i32 7
+ %lb7 = load float, ptr addrspace(3) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+
+define float @dot8_ordered_noacc(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b) {
+; GFX90A-LABEL: dot8_ordered_noacc:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[12:15], v[2:3], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[16:19], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[14:15]
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
+; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[6:7], v[18:19]
+; GFX90A-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
+; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[8:9], v[12:13]
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v7
+; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_ordered_noacc:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx4 v[4:7], v[2:3], off
+; GFX1030-NEXT: s_clause 0x1
+; GFX1030-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
+; GFX1030-NEXT: global_load_dwordx4 v[16:19], v[2:3], off offset:16
+; GFX1030-NEXT: s_waitcnt vmcnt(2)
+; GFX1030-NEXT: v_mul_f32_e32 v0, v9, v5
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v11, v7
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v15, v19
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_ordered_noacc:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[4:7], v[2:3], off
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_b128 v[8:11], v[0:1], off
+; GFX1250-NEXT: global_load_b128 v[12:15], v[0:1], off offset:16
+; GFX1250-NEXT: global_load_b128 v[16:19], v[2:3], off offset:16
+; GFX1250-NEXT: s_wait_loadcnt 0x2
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[8:9], v[4:5]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[10:11], v[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[14:15], v[18:19]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %s1 = fadd contract float %m0, %m1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+
+define float @dot8_sitofp_i16(ptr addrspace(1) noalias readonly align 16 %a, ptr addrspace(1) noalias readonly align 16 %b) {
+; GFX90A-LABEL: dot8_sitofp_i16:
+; GFX90A: ; %bb.0: ; %entry
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[2:3], off
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v13, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v12, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v14, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v17, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v16, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v19, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v18, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[14:15]
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[12:13], v[18:19]
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v9
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: dot8_sitofp_i16:
+; GFX1030: ; %bb.0: ; %entry
+; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx4 v[0:3], v[2:3], off
+; GFX1030-NEXT: s_waitcnt vmcnt(1)
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v8, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v9, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v4, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v10, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_mul_f32_e32 v0, v8, v9
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v8, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v9, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v4, v10
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v5, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v8, v9
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v8, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v2, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v4, v1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v1, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v4, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v5, v8
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v5, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v1, v2
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v1, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_cvt_f32_i32_sdwa v2, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v4, v5
+; GFX1030-NEXT: v_fmac_f32_e32 v0, v1, v2
+; GFX1030-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: dot8_sitofp_i16:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX1250-NEXT: global_load_b128 v[8:11], v[2:3], off
+; GFX1250-NEXT: s_wait_loadcnt 0x1
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_ashrrev_i32_e32 v0, 16, v4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 16, v8
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_bfe_i32 v2, v4, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v3, v8, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v4, v5, 0, 16
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX1250-NEXT: v_bfe_i32 v8, v9, 0, 16
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX1250-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 16, v5
+; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 16, v9
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v8, v8
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v2, v3
+; GFX1250-NEXT: v_bfe_i32 v2, v6, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v3, v10, 0, 16
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v5
+; GFX1250-NEXT: v_dual_fmac_f32 v0, v4, v8 :: v_dual_ashrrev_i32 v4, 16, v6
+; GFX1250-NEXT: v_ashrrev_i32_e32 v6, 16, v10
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v1, v5
+; GFX1250-NEXT: v_bfe_i32 v1, v7, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v5, v11, 0, 16
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v6
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v2, v3
+; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 16, v7
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_dual_fmac_f32 v0, v4, v6 :: v_dual_ashrrev_i32 v4, 16, v11
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v1, v3
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v2, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %pa0 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 0
+ %la0 = load i16, ptr addrspace(1) %pa0, align 16
+ %fa0 = sitofp i16 %la0 to float
+ %pb0 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 0
+ %lb0 = load i16, ptr addrspace(1) %pb0, align 16
+ %fb0 = sitofp i16 %lb0 to float
+ %m0 = fmul contract float %fa0, %fb0
+ %pa1 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 1
+ %la1 = load i16, ptr addrspace(1) %pa1, align 2
+ %fa1 = sitofp i16 %la1 to float
+ %pb1 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 1
+ %lb1 = load i16, ptr addrspace(1) %pb1, align 2
+ %fb1 = sitofp i16 %lb1 to float
+ %m1 = fmul contract float %fa1, %fb1
+ %s1 = fadd contract float %m0, %m1
+ %pa2 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 2
+ %la2 = load i16, ptr addrspace(1) %pa2, align 4
+ %fa2 = sitofp i16 %la2 to float
+ %pb2 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 2
+ %lb2 = load i16, ptr addrspace(1) %pb2, align 4
+ %fb2 = sitofp i16 %lb2 to float
+ %m2 = fmul contract float %fa2, %fb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 3
+ %la3 = load i16, ptr addrspace(1) %pa3, align 2
+ %fa3 = sitofp i16 %la3 to float
+ %pb3 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 3
+ %lb3 = load i16, ptr addrspace(1) %pb3, align 2
+ %fb3 = sitofp i16 %lb3 to float
+ %m3 = fmul contract float %fa3, %fb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 4
+ %la4 = load i16, ptr addrspace(1) %pa4, align 8
+ %fa4 = sitofp i16 %la4 to float
+ %pb4 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 4
+ %lb4 = load i16, ptr addrspace(1) %pb4, align 8
+ %fb4 = sitofp i16 %lb4 to float
+ %m4 = fmul contract float %fa4, %fb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 5
+ %la5 = load i16, ptr addrspace(1) %pa5, align 2
+ %fa5 = sitofp i16 %la5 to float
+ %pb5 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 5
+ %lb5 = load i16, ptr addrspace(1) %pb5, align 2
+ %fb5 = sitofp i16 %lb5 to float
+ %m5 = fmul contract float %fa5, %fb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 6
+ %la6 = load i16, ptr addrspace(1) %pa6, align 4
+ %fa6 = sitofp i16 %la6 to float
+ %pb6 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 6
+ %lb6 = load i16, ptr addrspace(1) %pb6, align 4
+ %fb6 = sitofp i16 %lb6 to float
+ %m6 = fmul contract float %fa6, %fb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 7
+ %la7 = load i16, ptr addrspace(1) %pa7, align 2
+ %fa7 = sitofp i16 %la7 to float
+ %pb7 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 7
+ %lb7 = load i16, ptr addrspace(1) %pb7, align 2
+ %fb7 = sitofp i16 %lb7 to float
+ %m7 = fmul contract float %fa7, %fb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
new file mode 100644
index 0000000000000..1948fe41f9af8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
@@ -0,0 +1,454 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX90A
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX942
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX950
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1030
+
+; Eight consecutive i8 weights are dequantized against a broadcast zero
+; point and multiplied into an ordered fadd chain. The scalar byte loads
+; already coalesce into one 64 bit load, so the byte loads and their
+; conversions stay scalar. The products stay scalar as well and fuse into
+; the fadd chain. The gfx9 targets subtract the zero point from four lanes
+; as a vector, gfx1030 keeps everything scalar.
+define float @dequant8_ordered(ptr addrspace(1) %mat, ptr addrspace(1) %vec, half %zph, float %acc) {
+; CHECK-LABEL: define float @dequant8_ordered(
+; CHECK-SAME: ptr addrspace(1) [[MAT:%.*]], ptr addrspace(1) [[VEC:%.*]], half [[ZPH:%.*]], float [[ACC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ZP:%.*]] = fpext half [[ZPH]] to float
+; CHECK-NEXT: br label %[[BODY:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x half>, ptr addrspace(1) [[VEC]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr addrspace(1) [[MAT]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[TMP0]] to <8 x float>
+; CHECK-NEXT: [[TMP3:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x float> poison, float [[ZP]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fsub contract <8 x float> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = fmul contract <8 x float> [[TMP6]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP7]], i64 0
+; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[ACC]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP7]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[A0]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP7]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[A1]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x float> [[TMP7]], i64 3
+; CHECK-NEXT: [[A3:%.*]] = fadd contract float [[A2]], [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[TMP7]], i64 4
+; CHECK-NEXT: [[A4:%.*]] = fadd contract float [[A3]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x float> [[TMP7]], i64 5
+; CHECK-NEXT: [[A5:%.*]] = fadd contract float [[A4]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x float> [[TMP7]], i64 6
+; CHECK-NEXT: [[A6:%.*]] = fadd contract float [[A5]], [[TMP14]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x float> [[TMP7]], i64 7
+; CHECK-NEXT: [[A7:%.*]] = fadd contract float [[A6]], [[TMP15]]
+; CHECK-NEXT: ret float [[A7]]
+;
+entry:
+ %zp = fpext half %zph to float
+ br label %body
+
+body:
+ %v0 = load half, ptr addrspace(1) %vec, align 16
+ %vp1 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 2
+ %v1 = load half, ptr addrspace(1) %vp1, align 2
+ %vp2 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 4
+ %v2 = load half, ptr addrspace(1) %vp2, align 4
+ %vp3 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 6
+ %v3 = load half, ptr addrspace(1) %vp3, align 2
+ %vp4 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 8
+ %v4 = load half, ptr addrspace(1) %vp4, align 8
+ %vp5 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 10
+ %v5 = load half, ptr addrspace(1) %vp5, align 2
+ %vp6 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 12
+ %v6 = load half, ptr addrspace(1) %vp6, align 4
+ %vp7 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 14
+ %v7 = load half, ptr addrspace(1) %vp7, align 2
+ %q0 = load i8, ptr addrspace(1) %mat, align 2
+ %mp1 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 1
+ %q1 = load i8, ptr addrspace(1) %mp1, align 1
+ %mp2 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 2
+ %q2 = load i8, ptr addrspace(1) %mp2, align 2
+ %mp3 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 3
+ %q3 = load i8, ptr addrspace(1) %mp3, align 1
+ %mp4 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 4
+ %q4 = load i8, ptr addrspace(1) %mp4, align 2
+ %mp5 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 5
+ %q5 = load i8, ptr addrspace(1) %mp5, align 1
+ %mp6 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 6
+ %q6 = load i8, ptr addrspace(1) %mp6, align 2
+ %mp7 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 7
+ %q7 = load i8, ptr addrspace(1) %mp7, align 1
+ %x0 = fpext half %v0 to float
+ %f0 = sitofp i8 %q0 to float
+ %d0 = fsub contract float %f0, %zp
+ %m0 = fmul contract float %d0, %x0
+ %a0 = fadd contract float %acc, %m0
+ %x1 = fpext half %v1 to float
+ %f1 = sitofp i8 %q1 to float
+ %d1 = fsub contract float %f1, %zp
+ %m1 = fmul contract float %d1, %x1
+ %a1 = fadd contract float %a0, %m1
+ %x2 = fpext half %v2 to float
+ %f2 = sitofp i8 %q2 to float
+ %d2 = fsub contract float %f2, %zp
+ %m2 = fmul contract float %d2, %x2
+ %a2 = fadd contract float %a1, %m2
+ %x3 = fpext half %v3 to float
+ %f3 = sitofp i8 %q3 to float
+ %d3 = fsub contract float %f3, %zp
+ %m3 = fmul contract float %d3, %x3
+ %a3 = fadd contract float %a2, %m3
+ %x4 = fpext half %v4 to float
+ %f4 = sitofp i8 %q4 to float
+ %d4 = fsub contract float %f4, %zp
+ %m4 = fmul contract float %d4, %x4
+ %a4 = fadd contract float %a3, %m4
+ %x5 = fpext half %v5 to float
+ %f5 = sitofp i8 %q5 to float
+ %d5 = fsub contract float %f5, %zp
+ %m5 = fmul contract float %d5, %x5
+ %a5 = fadd contract float %a4, %m5
+ %x6 = fpext half %v6 to float
+ %f6 = sitofp i8 %q6 to float
+ %d6 = fsub contract float %f6, %zp
+ %m6 = fmul contract float %d6, %x6
+ %a6 = fadd contract float %a5, %m6
+ %x7 = fpext half %v7 to float
+ %f7 = sitofp i8 %q7 to float
+ %d7 = fsub contract float %f7, %zp
+ %m7 = fmul contract float %d7, %x7
+ %a7 = fadd contract float %a6, %m7
+ ret float %a7
+}
+
+; A store chain over four converted bytes. The byte loads coalesce, so the
+; loads and the conversions stay scalar. gfx90a stores the converted values
+; with one vector store, gfx942 and gfx950 with two, and gfx1030 keeps the
+; scalar stores.
+define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: define void @store4_sitofp(
+; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 4
+; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+; GFX1030-LABEL: define void @store4_sitofp(
+; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX1030-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1
+; GFX1030-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2
+; GFX1030-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3
+; GFX1030-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 4
+; GFX1030-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1
+; GFX1030-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 2
+; GFX1030-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1
+; GFX1030-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float
+; GFX1030-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float
+; GFX1030-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float
+; GFX1030-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float
+; GFX1030-NEXT: [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
+; GFX1030-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
+; GFX1030-NEXT: [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
+; GFX1030-NEXT: store float [[F0]], ptr addrspace(1) [[OUT]], align 16
+; GFX1030-NEXT: store float [[F1]], ptr addrspace(1) [[O1]], align 4
+; GFX1030-NEXT: store float [[F2]], ptr addrspace(1) [[O2]], align 8
+; GFX1030-NEXT: store float [[F3]], ptr addrspace(1) [[O3]], align 4
+; GFX1030-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
+ %q0 = load i8, ptr addrspace(1) %in, align 4
+ %q1 = load i8, ptr addrspace(1) %p1, align 1
+ %q2 = load i8, ptr addrspace(1) %p2, align 2
+ %q3 = load i8, ptr addrspace(1) %p3, align 1
+ %f0 = sitofp i8 %q0 to float
+ %f1 = sitofp i8 %q1 to float
+ %f2 = sitofp i8 %q2 to float
+ %f3 = sitofp i8 %q3 to float
+ %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
+ store float %f0, ptr addrspace(1) %out, align 16
+ store float %f1, ptr addrspace(1) %o1, align 4
+ store float %f2, ptr addrspace(1) %o2, align 8
+ store float %f3, ptr addrspace(1) %o3, align 4
+ ret void
+}
+
+; The same store chain with every byte load aligned to one byte. Global loads
+; coalesce at any alignment, so the loads and the conversions stay scalar.
+define void @store4_sitofp_global_a1(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: define void @store4_sitofp_global_a1(
+; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 1
+; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+; GFX1030-LABEL: define void @store4_sitofp_global_a1(
+; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX1030-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1
+; GFX1030-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2
+; GFX1030-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3
+; GFX1030-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 1
+; GFX1030-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1
+; GFX1030-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 1
+; GFX1030-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1
+; GFX1030-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float
+; GFX1030-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float
+; GFX1030-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float
+; GFX1030-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float
+; GFX1030-NEXT: [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
+; GFX1030-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
+; GFX1030-NEXT: [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
+; GFX1030-NEXT: store float [[F0]], ptr addrspace(1) [[OUT]], align 16
+; GFX1030-NEXT: store float [[F1]], ptr addrspace(1) [[O1]], align 4
+; GFX1030-NEXT: store float [[F2]], ptr addrspace(1) [[O2]], align 8
+; GFX1030-NEXT: store float [[F3]], ptr addrspace(1) [[O3]], align 4
+; GFX1030-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
+ %q0 = load i8, ptr addrspace(1) %in, align 1
+ %q1 = load i8, ptr addrspace(1) %p1, align 1
+ %q2 = load i8, ptr addrspace(1) %p2, align 1
+ %q3 = load i8, ptr addrspace(1) %p3, align 1
+ %f0 = sitofp i8 %q0 to float
+ %f1 = sitofp i8 %q1 to float
+ %f2 = sitofp i8 %q2 to float
+ %f3 = sitofp i8 %q3 to float
+ %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
+ store float %f0, ptr addrspace(1) %out, align 16
+ store float %f1, ptr addrspace(1) %o1, align 4
+ store float %f2, ptr addrspace(1) %o2, align 8
+ store float %f3, ptr addrspace(1) %o3, align 4
+ ret void
+}
+
+; The same store chain over LDS with the first byte load aligned to four. The
+; byte loads coalesce into one dword load, so the loads and the conversions
+; stay scalar.
+define void @store4_sitofp_lds_a4(ptr addrspace(3) %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: define void @store4_sitofp_lds_a4(
+; GFX9-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(3) [[IN]], align 4
+; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+; GFX1030-LABEL: define void @store4_sitofp_lds_a4(
+; GFX1030-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX1030-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1
+; GFX1030-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2
+; GFX1030-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3
+; GFX1030-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 4
+; GFX1030-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1
+; GFX1030-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 2
+; GFX1030-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1
+; GFX1030-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float
+; GFX1030-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float
+; GFX1030-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float
+; GFX1030-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float
+; GFX1030-NEXT: [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
+; GFX1030-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
+; GFX1030-NEXT: [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
+; GFX1030-NEXT: store float [[F0]], ptr addrspace(1) [[OUT]], align 16
+; GFX1030-NEXT: store float [[F1]], ptr addrspace(1) [[O1]], align 4
+; GFX1030-NEXT: store float [[F2]], ptr addrspace(1) [[O2]], align 8
+; GFX1030-NEXT: store float [[F3]], ptr addrspace(1) [[O3]], align 4
+; GFX1030-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 3
+ %q0 = load i8, ptr addrspace(3) %in, align 4
+ %q1 = load i8, ptr addrspace(3) %p1, align 1
+ %q2 = load i8, ptr addrspace(3) %p2, align 2
+ %q3 = load i8, ptr addrspace(3) %p3, align 1
+ %f0 = sitofp i8 %q0 to float
+ %f1 = sitofp i8 %q1 to float
+ %f2 = sitofp i8 %q2 to float
+ %f3 = sitofp i8 %q3 to float
+ %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
+ store float %f0, ptr addrspace(1) %out, align 16
+ store float %f1, ptr addrspace(1) %o1, align 4
+ store float %f2, ptr addrspace(1) %o2, align 8
+ store float %f3, ptr addrspace(1) %o3, align 4
+ ret void
+}
+
+; The same store chain over LDS with every byte load aligned to one byte. The
+; backend does not merge these byte loads, so the vector load keeps its saving.
+; The gfx9 targets load, convert and store as vectors, gfx1030 keeps the scalar
+; form.
+define void @store4_sitofp_lds_a1(ptr addrspace(3) %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: define void @store4_sitofp_lds_a1(
+; GFX9-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(3) [[IN]], align 1
+; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+; GFX1030-LABEL: define void @store4_sitofp_lds_a1(
+; GFX1030-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GFX1030-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1
+; GFX1030-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2
+; GFX1030-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3
+; GFX1030-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 1
+; GFX1030-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1
+; GFX1030-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 1
+; GFX1030-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1
+; GFX1030-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float
+; GFX1030-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float
+; GFX1030-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float
+; GFX1030-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float
+; GFX1030-NEXT: [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
+; GFX1030-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
+; GFX1030-NEXT: [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
+; GFX1030-NEXT: store float [[F0]], ptr addrspace(1) [[OUT]], align 16
+; GFX1030-NEXT: store float [[F1]], ptr addrspace(1) [[O1]], align 4
+; GFX1030-NEXT: store float [[F2]], ptr addrspace(1) [[O2]], align 8
+; GFX1030-NEXT: store float [[F3]], ptr addrspace(1) [[O3]], align 4
+; GFX1030-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 3
+ %q0 = load i8, ptr addrspace(3) %in, align 1
+ %q1 = load i8, ptr addrspace(3) %p1, align 1
+ %q2 = load i8, ptr addrspace(3) %p2, align 1
+ %q3 = load i8, ptr addrspace(3) %p3, align 1
+ %f0 = sitofp i8 %q0 to float
+ %f1 = sitofp i8 %q1 to float
+ %f2 = sitofp i8 %q2 to float
+ %f3 = sitofp i8 %q3 to float
+ %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
+ store float %f0, ptr addrspace(1) %out, align 16
+ store float %f1, ptr addrspace(1) %o1, align 4
+ store float %f2, ptr addrspace(1) %o2, align 8
+ store float %f3, ptr addrspace(1) %o3, align 4
+ ret void
+}
+
+; Four zero extended bytes are inserted into a vector that is stored, inside a
+; loop of 16 iterations. The byte loads coalesce, and the load saving that never
+; materializes is cancelled in every iteration, so the loop body stays scalar.
+define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; CHECK-LABEL: define void @zext4_bv_loop(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[OFF:%.*]] = shl i64 [[I]], 2
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[OFF]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[P0]], align 4
+; CHECK-NEXT: [[V3:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[PO:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[OUT]], i64 [[I]]
+; CHECK-NEXT: store <4 x i32> [[V3]], ptr addrspace(1) [[PO]], align 16
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], 16
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %off = shl i64 %i, 2
+ %p0 = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %off
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 3
+ %l0 = load i8, ptr addrspace(1) %p0, align 4
+ %l1 = load i8, ptr addrspace(1) %p1, align 1
+ %l2 = load i8, ptr addrspace(1) %p2, align 2
+ %l3 = load i8, ptr addrspace(1) %p3, align 1
+ %x0 = zext i8 %l0 to i32
+ %x1 = zext i8 %l1 to i32
+ %x2 = zext i8 %l2 to i32
+ %x3 = zext i8 %l3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ %po = getelementptr inbounds <4 x i32>, ptr addrspace(1) %out, i64 %i
+ store <4 x i32> %v3, ptr addrspace(1) %po, align 16
+ %i.next = add nuw nsw i64 %i, 1
+ %c = icmp ult i64 %i.next, 16
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
+
+; Eight bytes are converted to half and stored. Two half lanes fit one
+; register, so the byte loads keep their saving and the loads, the conversions
+; and the stores stay vectorized. On gfx1250 the scalar form takes 37
+; instructions against 27 for the vector form.
+define void @store8_sitofp_half(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; CHECK-LABEL: define void @store8_sitofp_half(
+; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 4
+; CHECK-NEXT: [[O4:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[OUT]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x half>
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr addrspace(1) [[P4]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = sitofp <4 x i8> [[TMP3]] to <4 x half>
+; CHECK-NEXT: store <4 x half> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
+; CHECK-NEXT: store <4 x half> [[TMP4]], ptr addrspace(1) [[O4]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 4
+ %p5 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 5
+ %p6 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 6
+ %p7 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 7
+ %q0 = load i8, ptr addrspace(1) %in, align 8
+ %q1 = load i8, ptr addrspace(1) %p1, align 1
+ %q2 = load i8, ptr addrspace(1) %p2, align 2
+ %q3 = load i8, ptr addrspace(1) %p3, align 1
+ %q4 = load i8, ptr addrspace(1) %p4, align 4
+ %q5 = load i8, ptr addrspace(1) %p5, align 1
+ %q6 = load i8, ptr addrspace(1) %p6, align 2
+ %q7 = load i8, ptr addrspace(1) %p7, align 1
+ %f0 = sitofp i8 %q0 to half
+ %f1 = sitofp i8 %q1 to half
+ %f2 = sitofp i8 %q2 to half
+ %f3 = sitofp i8 %q3 to half
+ %f4 = sitofp i8 %q4 to half
+ %f5 = sitofp i8 %q5 to half
+ %f6 = sitofp i8 %q6 to half
+ %f7 = sitofp i8 %q7 to half
+ %o1 = getelementptr inbounds half, ptr addrspace(1) %out, i64 1
+ %o2 = getelementptr inbounds half, ptr addrspace(1) %out, i64 2
+ %o3 = getelementptr inbounds half, ptr addrspace(1) %out, i64 3
+ %o4 = getelementptr inbounds half, ptr addrspace(1) %out, i64 4
+ %o5 = getelementptr inbounds half, ptr addrspace(1) %out, i64 5
+ %o6 = getelementptr inbounds half, ptr addrspace(1) %out, i64 6
+ %o7 = getelementptr inbounds half, ptr addrspace(1) %out, i64 7
+ store half %f0, ptr addrspace(1) %out, align 16
+ store half %f1, ptr addrspace(1) %o1, align 2
+ store half %f2, ptr addrspace(1) %o2, align 4
+ store half %f3, ptr addrspace(1) %o3, align 2
+ store half %f4, ptr addrspace(1) %o4, align 8
+ store half %f5, ptr addrspace(1) %o5, align 2
+ store half %f6, ptr addrspace(1) %o6, align 4
+ store half %f7, ptr addrspace(1) %o7, align 2
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX90A: {{.*}}
+; GFX942: {{.*}}
+; GFX942-950: {{.*}}
+; GFX950: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-reduction.ll
new file mode 100644
index 0000000000000..27079d00a1733
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-reduction.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=GFX90A
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefixes=GFX1030
+
+; Four approximate divisions of sums, added up by a reassociable reduction.
+; The vector fdiv is the root of the tree and keeps the approximation flag of
+; its scalar divisions, which the target lowers to a reciprocal and a multiply
+; per lane, so the divisions and the sums feeding them vectorize.
+define float @div4_afn_sum(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b, float %k, float %acc) {
+; GFX90A-LABEL: define float @div4_afn_sum(
+; GFX90A-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]], float [[K:%.*]], float [[ACC:%.*]]) {
+; GFX90A-NEXT: [[ENTRY:.*:]]
+; GFX90A-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; GFX90A-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; GFX90A-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[PA0]], align 4
+; GFX90A-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[PB0]], align 4
+; GFX90A-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[K]], i64 0
+; GFX90A-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> zeroinitializer
+; GFX90A-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP0]], [[TMP3]]
+; GFX90A-NEXT: [[TMP5:%.*]] = fdiv afn <4 x float> [[TMP4]], [[TMP1]]
+; GFX90A-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP5]])
+; GFX90A-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP6]], [[ACC]]
+; GFX90A-NEXT: ret float [[OP_RDX]]
+;
+; GFX1030-LABEL: define float @div4_afn_sum(
+; GFX1030-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]], float [[K:%.*]], float [[ACC:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; GFX1030-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[PA0]], align 4
+; GFX1030-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[PB0]], align 4
+; GFX1030-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[K]], i64 0
+; GFX1030-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> zeroinitializer
+; GFX1030-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP0]], [[TMP3]]
+; GFX1030-NEXT: [[TMP5:%.*]] = fdiv afn <4 x float> [[TMP4]], [[TMP1]]
+; GFX1030-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP5]])
+; GFX1030-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP6]], [[ACC]]
+; GFX1030-NEXT: ret float [[OP_RDX]]
+;
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %t0 = fadd float %la0, %k
+ %d0 = fdiv afn float %t0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %t1 = fadd float %la1, %k
+ %d1 = fdiv afn float %t1, %lb1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %t2 = fadd float %la2, %k
+ %d2 = fdiv afn float %t2, %lb2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %t3 = fadd float %la3, %k
+ %d3 = fdiv afn float %t3, %lb3
+ %s0 = fadd fast float %acc, %d0
+ %s1 = fadd fast float %s0, %d1
+ %s2 = fadd fast float %s1, %d2
+ %s3 = fadd fast float %s2, %d3
+ ret float %s3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll
new file mode 100644
index 0000000000000..6a178d20eb717
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll
@@ -0,0 +1,527 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX90A
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX942
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX950
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1030
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1250
+
+; Integer reductions over consecutive byte loads that are widened before
+; the arithmetic. The scalar byte loads already coalesce into one wide
+; load per pointer, so a vector load bundle saves nothing, and once that
+; saving is charged neither the dot product nor the sum is profitable.
+; Vectorized on gfx90a, the dot product of eight sign extended bytes grew
+; from 8 to 23 instructions and the sum of four zero extended bytes from
+; 8 to 14. The and reduction stays vectorized because its zero extension
+; folds away and the vector form works on the bytes as loaded.
+; Lanes of 16 bits are packed two to a register, so the sum into i16 lanes
+; and the dot product of i16 loads keep their saving and stay vectorized.
+
+define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; CHECK-LABEL: define i32 @dot8_sext_i8(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr addrspace(1) [[PB0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 4
+; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA4]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i8> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i8>, ptr addrspace(1) [[PB4]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[R7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX_OP]])
+; CHECK-NEXT: ret i32 [[R7]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = sext i8 %la0 to i32
+ %pb0 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 0
+ %lb0 = load i8, ptr addrspace(1) %pb0, align 1
+ %xb0 = sext i8 %lb0 to i32
+ %m0 = mul nsw i32 %xa0, %xb0
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = sext i8 %la1 to i32
+ %pb1 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 1
+ %lb1 = load i8, ptr addrspace(1) %pb1, align 1
+ %xb1 = sext i8 %lb1 to i32
+ %m1 = mul nsw i32 %xa1, %xb1
+ %r1 = add nsw i32 %m0, %m1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = sext i8 %la2 to i32
+ %pb2 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 2
+ %lb2 = load i8, ptr addrspace(1) %pb2, align 1
+ %xb2 = sext i8 %lb2 to i32
+ %m2 = mul nsw i32 %xa2, %xb2
+ %r2 = add nsw i32 %r1, %m2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = sext i8 %la3 to i32
+ %pb3 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 3
+ %lb3 = load i8, ptr addrspace(1) %pb3, align 1
+ %xb3 = sext i8 %lb3 to i32
+ %m3 = mul nsw i32 %xa3, %xb3
+ %r3 = add nsw i32 %r2, %m3
+ %pa4 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 4
+ %la4 = load i8, ptr addrspace(1) %pa4, align 1
+ %xa4 = sext i8 %la4 to i32
+ %pb4 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 4
+ %lb4 = load i8, ptr addrspace(1) %pb4, align 1
+ %xb4 = sext i8 %lb4 to i32
+ %m4 = mul nsw i32 %xa4, %xb4
+ %r4 = add nsw i32 %r3, %m4
+ %pa5 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 5
+ %la5 = load i8, ptr addrspace(1) %pa5, align 1
+ %xa5 = sext i8 %la5 to i32
+ %pb5 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 5
+ %lb5 = load i8, ptr addrspace(1) %pb5, align 1
+ %xb5 = sext i8 %lb5 to i32
+ %m5 = mul nsw i32 %xa5, %xb5
+ %r5 = add nsw i32 %r4, %m5
+ %pa6 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 6
+ %la6 = load i8, ptr addrspace(1) %pa6, align 1
+ %xa6 = sext i8 %la6 to i32
+ %pb6 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 6
+ %lb6 = load i8, ptr addrspace(1) %pb6, align 1
+ %xb6 = sext i8 %lb6 to i32
+ %m6 = mul nsw i32 %xa6, %xb6
+ %r6 = add nsw i32 %r5, %m6
+ %pa7 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 7
+ %la7 = load i8, ptr addrspace(1) %pa7, align 1
+ %xa7 = sext i8 %la7 to i32
+ %pb7 = getelementptr inbounds i8, ptr addrspace(1) %b, i32 7
+ %lb7 = load i8, ptr addrspace(1) %pb7, align 1
+ %xb7 = sext i8 %lb7 to i32
+ %m7 = mul nsw i32 %xa7, %xb7
+ %r7 = add nsw i32 %r6, %m7
+ ret i32 %r7
+}
+
+define i32 @sum4_zext_i8(ptr addrspace(1) %a) {
+; CHECK-LABEL: define i32 @sum4_zext_i8(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[R3]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = zext i8 %la0 to i32
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = zext i8 %la1 to i32
+ %r1 = add nsw i32 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = zext i8 %la2 to i32
+ %r2 = add nsw i32 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = zext i8 %la3 to i32
+ %r3 = add nsw i32 %r2, %xa3
+ ret i32 %r3
+}
+
+define i32 @and8_zext_i8(ptr addrspace(1) %a) {
+; CHECK-LABEL: define i32 @and8_zext_i8(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr addrspace(1) [[P0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT: ret i32 [[TMP2]]
+;
+entry:
+ %p0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %l0 = load i8, ptr addrspace(1) %p0, align 1
+ %x0 = zext i8 %l0 to i32
+ %p1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %l1 = load i8, ptr addrspace(1) %p1, align 1
+ %x1 = zext i8 %l1 to i32
+ %p2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %l2 = load i8, ptr addrspace(1) %p2, align 1
+ %x2 = zext i8 %l2 to i32
+ %p3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %l3 = load i8, ptr addrspace(1) %p3, align 1
+ %x3 = zext i8 %l3 to i32
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 4
+ %l4 = load i8, ptr addrspace(1) %p4, align 1
+ %x4 = zext i8 %l4 to i32
+ %p5 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 5
+ %l5 = load i8, ptr addrspace(1) %p5, align 1
+ %x5 = zext i8 %l5 to i32
+ %p6 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 6
+ %l6 = load i8, ptr addrspace(1) %p6, align 1
+ %x6 = zext i8 %l6 to i32
+ %p7 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 7
+ %l7 = load i8, ptr addrspace(1) %p7, align 1
+ %x7 = zext i8 %l7 to i32
+ %r1 = and i32 %x0, %x1
+ %r2 = and i32 %r1, %x2
+ %r3 = and i32 %r2, %x3
+ %r4 = and i32 %r3, %x4
+ %r5 = and i32 %r4, %x5
+ %r6 = and i32 %r5, %x6
+ %r7 = and i32 %r6, %x7
+ ret i32 %r7
+}
+
+; The sum of four zero extended bytes inside a loop of 16 iterations. The load
+; saving is cancelled in every iteration, so the sum stays scalar there too.
+define void @sum4_zext_i8_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; CHECK-LABEL: define void @sum4_zext_i8_loop(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[OFF:%.*]] = shl i32 [[I]], 2
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 [[OFF]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[PO:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i32 [[I]]
+; CHECK-NEXT: store i32 [[R3]], ptr addrspace(1) [[PO]], align 4
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], 16
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %off = shl i32 %i, 2
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 %off
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = zext i8 %la0 to i32
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %pa0, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = zext i8 %la1 to i32
+ %r1 = add nsw i32 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %pa0, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = zext i8 %la2 to i32
+ %r2 = add nsw i32 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %pa0, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = zext i8 %la3 to i32
+ %r3 = add nsw i32 %r2, %xa3
+ %po = getelementptr inbounds i32, ptr addrspace(1) %out, i32 %i
+ store i32 %r3, ptr addrspace(1) %po, align 4
+ %i.next = add nuw nsw i32 %i, 1
+ %c = icmp ult i32 %i.next, 16
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
+
+; The same sum over LDS with the first byte load aligned to four. The byte
+; loads coalesce into one dword load, so the sum stays scalar.
+define i32 @sum4_zext_i8_lds_a4(ptr addrspace(3) %a) {
+; CHECK-LABEL: define i32 @sum4_zext_i8_lds_a4(
+; CHECK-SAME: ptr addrspace(3) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[R3]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 0
+ %la0 = load i8, ptr addrspace(3) %pa0, align 4
+ %xa0 = zext i8 %la0 to i32
+ %pa1 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 1
+ %la1 = load i8, ptr addrspace(3) %pa1, align 1
+ %xa1 = zext i8 %la1 to i32
+ %r1 = add nsw i32 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 2
+ %la2 = load i8, ptr addrspace(3) %pa2, align 2
+ %xa2 = zext i8 %la2 to i32
+ %r2 = add nsw i32 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 3
+ %la3 = load i8, ptr addrspace(3) %pa3, align 1
+ %xa3 = zext i8 %la3 to i32
+ %r3 = add nsw i32 %r2, %xa3
+ ret i32 %r3
+}
+
+; The same sum over LDS with every byte load aligned to one byte. The backend
+; keeps these byte loads apart, so their load saving stays and the sum is
+; vectorized. The vector form reads the bytes through one misaligned dword
+; load and takes 14 or 15 instructions against 10 for the scalar form.
+define i32 @sum4_zext_i8_lds_a1(ptr addrspace(3) %a) {
+; CHECK-LABEL: define i32 @sum4_zext_i8_lds_a1(
+; CHECK-SAME: ptr addrspace(3) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 0
+ %la0 = load i8, ptr addrspace(3) %pa0, align 1
+ %xa0 = zext i8 %la0 to i32
+ %pa1 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 1
+ %la1 = load i8, ptr addrspace(3) %pa1, align 1
+ %xa1 = zext i8 %la1 to i32
+ %r1 = add nsw i32 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 2
+ %la2 = load i8, ptr addrspace(3) %pa2, align 1
+ %xa2 = zext i8 %la2 to i32
+ %r2 = add nsw i32 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 3
+ %la3 = load i8, ptr addrspace(3) %pa3, align 1
+ %xa3 = zext i8 %la3 to i32
+ %r3 = add nsw i32 %r2, %xa3
+ ret i32 %r3
+}
+
+; The dot product of eight sign extended bytes over LDS with the first byte
+; load of each pointer aligned to 16. The byte loads coalesce into one load
+; per pointer, and a bundle of them is judged by the best alignment among
+; its loads, so a bundle starting at the second byte coalesces as well. The
+; dot product stays scalar. Vectorized on gfx90a it took 23 instructions
+; against 8 for the scalar form.
+define i32 @dot8_sext_i8_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b) {
+; CHECK-LABEL: define i32 @dot8_sext_i8_lds_a16(
+; CHECK-SAME: ptr addrspace(3) [[A:%.*]], ptr addrspace(3) [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA0]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr addrspace(3) [[PB0]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 4
+; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA4]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i8> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i8>, ptr addrspace(3) [[PB4]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX_OP]])
+; CHECK-NEXT: ret i32 [[TMP10]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 0
+ %la0 = load i8, ptr addrspace(3) %pa0, align 16
+ %xa0 = sext i8 %la0 to i32
+ %pb0 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 0
+ %lb0 = load i8, ptr addrspace(3) %pb0, align 16
+ %xb0 = sext i8 %lb0 to i32
+ %m0 = mul nsw i32 %xa0, %xb0
+ %pa1 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 1
+ %la1 = load i8, ptr addrspace(3) %pa1, align 1
+ %xa1 = sext i8 %la1 to i32
+ %pb1 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 1
+ %lb1 = load i8, ptr addrspace(3) %pb1, align 1
+ %xb1 = sext i8 %lb1 to i32
+ %m1 = mul nsw i32 %xa1, %xb1
+ %r1 = add nsw i32 %m0, %m1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 2
+ %la2 = load i8, ptr addrspace(3) %pa2, align 2
+ %xa2 = sext i8 %la2 to i32
+ %pb2 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 2
+ %lb2 = load i8, ptr addrspace(3) %pb2, align 2
+ %xb2 = sext i8 %lb2 to i32
+ %m2 = mul nsw i32 %xa2, %xb2
+ %r2 = add nsw i32 %r1, %m2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 3
+ %la3 = load i8, ptr addrspace(3) %pa3, align 1
+ %xa3 = sext i8 %la3 to i32
+ %pb3 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 3
+ %lb3 = load i8, ptr addrspace(3) %pb3, align 1
+ %xb3 = sext i8 %lb3 to i32
+ %m3 = mul nsw i32 %xa3, %xb3
+ %r3 = add nsw i32 %r2, %m3
+ %pa4 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 4
+ %la4 = load i8, ptr addrspace(3) %pa4, align 4
+ %xa4 = sext i8 %la4 to i32
+ %pb4 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 4
+ %lb4 = load i8, ptr addrspace(3) %pb4, align 4
+ %xb4 = sext i8 %lb4 to i32
+ %m4 = mul nsw i32 %xa4, %xb4
+ %r4 = add nsw i32 %r3, %m4
+ %pa5 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 5
+ %la5 = load i8, ptr addrspace(3) %pa5, align 1
+ %xa5 = sext i8 %la5 to i32
+ %pb5 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 5
+ %lb5 = load i8, ptr addrspace(3) %pb5, align 1
+ %xb5 = sext i8 %lb5 to i32
+ %m5 = mul nsw i32 %xa5, %xb5
+ %r5 = add nsw i32 %r4, %m5
+ %pa6 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 6
+ %la6 = load i8, ptr addrspace(3) %pa6, align 2
+ %xa6 = sext i8 %la6 to i32
+ %pb6 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 6
+ %lb6 = load i8, ptr addrspace(3) %pb6, align 2
+ %xb6 = sext i8 %lb6 to i32
+ %m6 = mul nsw i32 %xa6, %xb6
+ %r6 = add nsw i32 %r5, %m6
+ %pa7 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 7
+ %la7 = load i8, ptr addrspace(3) %pa7, align 1
+ %xa7 = sext i8 %la7 to i32
+ %pb7 = getelementptr inbounds i8, ptr addrspace(3) %b, i32 7
+ %lb7 = load i8, ptr addrspace(3) %pb7, align 1
+ %xb7 = sext i8 %lb7 to i32
+ %m7 = mul nsw i32 %xa7, %xb7
+ %r7 = add nsw i32 %r6, %m7
+ ret i32 %r7
+}
+
+; The sum of four sign extended bytes into i16 lanes. Two i16 lanes fit one
+; register, so the byte loads keep their saving and the sum stays vectorized.
+; On gfx1250 the scalar form takes 24 instructions against 18 for the vector
+; form.
+define i16 @sum4_sext_i8_i16(ptr addrspace(1) %a) {
+; CHECK-LABEL: define i16 @sum4_sext_i8_i16(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: ret i16 [[TMP2]]
+;
+entry:
+ %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
+ %la0 = load i8, ptr addrspace(1) %pa0, align 1
+ %xa0 = sext i8 %la0 to i16
+ %pa1 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 1
+ %la1 = load i8, ptr addrspace(1) %pa1, align 1
+ %xa1 = sext i8 %la1 to i16
+ %r1 = add nsw i16 %xa0, %xa1
+ %pa2 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 2
+ %la2 = load i8, ptr addrspace(1) %pa2, align 1
+ %xa2 = sext i8 %la2 to i16
+ %r2 = add nsw i16 %r1, %xa2
+ %pa3 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 3
+ %la3 = load i8, ptr addrspace(1) %pa3, align 1
+ %xa3 = sext i8 %la3 to i16
+ %r3 = add nsw i16 %r2, %xa3
+ ret i16 %r3
+}
+
+; The dot product of eight zero extended i16 loads. The i16 loads keep their
+; saving and the dot product stays vectorized. On gfx1250 the vector form
+; saves three VALU instructions, the gfx9 and gfx10 targets spend one more.
+define i32 @dot8_zext_i16(ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; CHECK-LABEL: define i32 @dot8_zext_i16(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i32 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[PA0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i16>, ptr addrspace(1) [[PB0]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i32 4
+; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i16>, ptr addrspace(1) [[PA4]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i16>, ptr addrspace(1) [[PB4]], align 2
+; CHECK-NEXT: [[TMP8:%.*]] = zext <4 x i16> [[TMP7]] to <4 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX_OP]])
+; CHECK-NEXT: ret i32 [[TMP10]]
+;
+entry:
+ %pa0 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 0
+ %la0 = load i16, ptr addrspace(1) %pa0, align 2
+ %xa0 = zext i16 %la0 to i32
+ %pb0 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 0
+ %lb0 = load i16, ptr addrspace(1) %pb0, align 2
+ %xb0 = zext i16 %lb0 to i32
+ %m0 = mul nsw i32 %xa0, %xb0
+ %pa1 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 1
+ %la1 = load i16, ptr addrspace(1) %pa1, align 2
+ %xa1 = zext i16 %la1 to i32
+ %pb1 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 1
+ %lb1 = load i16, ptr addrspace(1) %pb1, align 2
+ %xb1 = zext i16 %lb1 to i32
+ %m1 = mul nsw i32 %xa1, %xb1
+ %r1 = add nsw i32 %m0, %m1
+ %pa2 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 2
+ %la2 = load i16, ptr addrspace(1) %pa2, align 2
+ %xa2 = zext i16 %la2 to i32
+ %pb2 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 2
+ %lb2 = load i16, ptr addrspace(1) %pb2, align 2
+ %xb2 = zext i16 %lb2 to i32
+ %m2 = mul nsw i32 %xa2, %xb2
+ %r2 = add nsw i32 %r1, %m2
+ %pa3 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 3
+ %la3 = load i16, ptr addrspace(1) %pa3, align 2
+ %xa3 = zext i16 %la3 to i32
+ %pb3 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 3
+ %lb3 = load i16, ptr addrspace(1) %pb3, align 2
+ %xb3 = zext i16 %lb3 to i32
+ %m3 = mul nsw i32 %xa3, %xb3
+ %r3 = add nsw i32 %r2, %m3
+ %pa4 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 4
+ %la4 = load i16, ptr addrspace(1) %pa4, align 2
+ %xa4 = zext i16 %la4 to i32
+ %pb4 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 4
+ %lb4 = load i16, ptr addrspace(1) %pb4, align 2
+ %xb4 = zext i16 %lb4 to i32
+ %m4 = mul nsw i32 %xa4, %xb4
+ %r4 = add nsw i32 %r3, %m4
+ %pa5 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 5
+ %la5 = load i16, ptr addrspace(1) %pa5, align 2
+ %xa5 = zext i16 %la5 to i32
+ %pb5 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 5
+ %lb5 = load i16, ptr addrspace(1) %pb5, align 2
+ %xb5 = zext i16 %lb5 to i32
+ %m5 = mul nsw i32 %xa5, %xb5
+ %r5 = add nsw i32 %r4, %m5
+ %pa6 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 6
+ %la6 = load i16, ptr addrspace(1) %pa6, align 2
+ %xa6 = zext i16 %la6 to i32
+ %pb6 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 6
+ %lb6 = load i16, ptr addrspace(1) %pb6, align 2
+ %xb6 = zext i16 %lb6 to i32
+ %m6 = mul nsw i32 %xa6, %xb6
+ %r6 = add nsw i32 %r5, %m6
+ %pa7 = getelementptr inbounds i16, ptr addrspace(1) %a, i32 7
+ %la7 = load i16, ptr addrspace(1) %pa7, align 2
+ %xa7 = zext i16 %la7 to i32
+ %pb7 = getelementptr inbounds i16, ptr addrspace(1) %b, i32 7
+ %lb7 = load i16, ptr addrspace(1) %pb7, align 2
+ %xb7 = zext i16 %lb7 to i32
+ %m7 = mul nsw i32 %xa7, %xb7
+ %r7 = add nsw i32 %r6, %m7
+ ret i32 %r7
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1030: {{.*}}
+; GFX1250: {{.*}}
+; GFX9: {{.*}}
+; GFX90A: {{.*}}
+; GFX942: {{.*}}
+; GFX942-950: {{.*}}
+; GFX950: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
new file mode 100644
index 0000000000000..f7f2f69c1366e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
@@ -0,0 +1,684 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX90A
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX942
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9,GFX942-950,GFX950
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1030
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1250
+
+; Ordered contract fadd reduction over consecutive global loads. On AMDGPU
+; the scalar loads coalesce into the same wide access a vector load would
+; use, so the reduction must stay scalar and fuse into fmas.
+
+define amdgpu_kernel void @dotconv(ptr addrspace(1) noalias readonly align 4 %in, ptr addrspace(1) noalias readonly align 4 %mask, ptr addrspace(1) noalias writeonly align 4 %out) {
+; CHECK-LABEL: define amdgpu_kernel void @dotconv(
+; CHECK-SAME: ptr addrspace(1) noalias readonly align 4 [[IN:%.*]], ptr addrspace(1) noalias readonly align 4 [[MASK:%.*]], ptr addrspace(1) noalias writeonly align 4 [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CALL_I:%.*]] = tail call i64 @_Z13get_global_idj(i32 noundef 0)
+; CHECK-NEXT: [[SEXT_I:%.*]] = shl i64 [[CALL_I]], 32
+; CHECK-NEXT: [[IDX_EXT_I:%.*]] = ashr exact i64 [[SEXT_I]], 32
+; CHECK-NEXT: [[ADD_PTR_I:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(1) [[IN]], i64 [[IDX_EXT_I]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr addrspace(1) [[ADD_PTR_I]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr addrspace(1) [[MASK]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <16 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[MUL_I:%.*]] = extractelement <16 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[ADD_I:%.*]] = fadd contract float [[MUL_I]], 0.000000e+00
+; CHECK-NEXT: [[MUL_I_1:%.*]] = extractelement <16 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[ADD_I_1:%.*]] = fadd contract float [[ADD_I]], [[MUL_I_1]]
+; CHECK-NEXT: [[MUL_I_2:%.*]] = extractelement <16 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[ADD_I_2:%.*]] = fadd contract float [[ADD_I_1]], [[MUL_I_2]]
+; CHECK-NEXT: [[MUL_I_3:%.*]] = extractelement <16 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[ADD_I_3:%.*]] = fadd contract float [[ADD_I_2]], [[MUL_I_3]]
+; CHECK-NEXT: [[MUL_I_4:%.*]] = extractelement <16 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[ADD_I_4:%.*]] = fadd contract float [[ADD_I_3]], [[MUL_I_4]]
+; CHECK-NEXT: [[MUL_I_5:%.*]] = extractelement <16 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[ADD_I_5:%.*]] = fadd contract float [[ADD_I_4]], [[MUL_I_5]]
+; CHECK-NEXT: [[MUL_I_6:%.*]] = extractelement <16 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[ADD_I_6:%.*]] = fadd contract float [[ADD_I_5]], [[MUL_I_6]]
+; CHECK-NEXT: [[MUL_I_7:%.*]] = extractelement <16 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[ADD_I_7:%.*]] = fadd contract float [[ADD_I_6]], [[MUL_I_7]]
+; CHECK-NEXT: [[MUL_I_8:%.*]] = extractelement <16 x float> [[TMP2]], i64 8
+; CHECK-NEXT: [[ADD_I_8:%.*]] = fadd contract float [[ADD_I_7]], [[MUL_I_8]]
+; CHECK-NEXT: [[MUL_I_9:%.*]] = extractelement <16 x float> [[TMP2]], i64 9
+; CHECK-NEXT: [[ADD_I_9:%.*]] = fadd contract float [[ADD_I_8]], [[MUL_I_9]]
+; CHECK-NEXT: [[MUL_I_10:%.*]] = extractelement <16 x float> [[TMP2]], i64 10
+; CHECK-NEXT: [[ADD_I_10:%.*]] = fadd contract float [[ADD_I_9]], [[MUL_I_10]]
+; CHECK-NEXT: [[MUL_I_11:%.*]] = extractelement <16 x float> [[TMP2]], i64 11
+; CHECK-NEXT: [[ADD_I_11:%.*]] = fadd contract float [[ADD_I_10]], [[MUL_I_11]]
+; CHECK-NEXT: [[MUL_I_12:%.*]] = extractelement <16 x float> [[TMP2]], i64 12
+; CHECK-NEXT: [[ADD_I_12:%.*]] = fadd contract float [[ADD_I_11]], [[MUL_I_12]]
+; CHECK-NEXT: [[MUL_I_13:%.*]] = extractelement <16 x float> [[TMP2]], i64 13
+; CHECK-NEXT: [[ADD_I_13:%.*]] = fadd contract float [[ADD_I_12]], [[MUL_I_13]]
+; CHECK-NEXT: [[MUL_I_14:%.*]] = extractelement <16 x float> [[TMP2]], i64 14
+; CHECK-NEXT: [[ADD_I_14:%.*]] = fadd contract float [[ADD_I_13]], [[MUL_I_14]]
+; CHECK-NEXT: [[MUL_I_15:%.*]] = extractelement <16 x float> [[TMP2]], i64 15
+; CHECK-NEXT: [[ADD_I_15:%.*]] = fadd contract float [[ADD_I_14]], [[MUL_I_15]]
+; CHECK-NEXT: [[ARRAYIDX5_I:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(1) [[OUT]], i64 [[IDX_EXT_I]]
+; CHECK-NEXT: store float [[ADD_I_15]], ptr addrspace(1) [[ARRAYIDX5_I]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %call.i = tail call i64 @_Z13get_global_idj(i32 noundef 0)
+ %sext.i = shl i64 %call.i, 32
+ %idx.ext.i = ashr exact i64 %sext.i, 32
+ %add.ptr.i = getelementptr inbounds [4 x i8], ptr addrspace(1) %in, i64 %idx.ext.i
+ %0 = load float, ptr addrspace(1) %add.ptr.i, align 4
+ %1 = load float, ptr addrspace(1) %mask, align 4
+ %mul.i = fmul contract float %0, %1
+ %add.i = fadd contract float %mul.i, 0.000000e+00
+ %arrayidx.i.1 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 4
+ %2 = load float, ptr addrspace(1) %arrayidx.i.1, align 4
+ %arrayidx3.i.1 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 4
+ %3 = load float, ptr addrspace(1) %arrayidx3.i.1, align 4
+ %mul.i.1 = fmul contract float %2, %3
+ %add.i.1 = fadd contract float %add.i, %mul.i.1
+ %arrayidx.i.2 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 8
+ %4 = load float, ptr addrspace(1) %arrayidx.i.2, align 4
+ %arrayidx3.i.2 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 8
+ %5 = load float, ptr addrspace(1) %arrayidx3.i.2, align 4
+ %mul.i.2 = fmul contract float %4, %5
+ %add.i.2 = fadd contract float %add.i.1, %mul.i.2
+ %arrayidx.i.3 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 12
+ %6 = load float, ptr addrspace(1) %arrayidx.i.3, align 4
+ %arrayidx3.i.3 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 12
+ %7 = load float, ptr addrspace(1) %arrayidx3.i.3, align 4
+ %mul.i.3 = fmul contract float %6, %7
+ %add.i.3 = fadd contract float %add.i.2, %mul.i.3
+ %arrayidx.i.4 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 16
+ %8 = load float, ptr addrspace(1) %arrayidx.i.4, align 4
+ %arrayidx3.i.4 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 16
+ %9 = load float, ptr addrspace(1) %arrayidx3.i.4, align 4
+ %mul.i.4 = fmul contract float %8, %9
+ %add.i.4 = fadd contract float %add.i.3, %mul.i.4
+ %arrayidx.i.5 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 20
+ %10 = load float, ptr addrspace(1) %arrayidx.i.5, align 4
+ %arrayidx3.i.5 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 20
+ %11 = load float, ptr addrspace(1) %arrayidx3.i.5, align 4
+ %mul.i.5 = fmul contract float %10, %11
+ %add.i.5 = fadd contract float %add.i.4, %mul.i.5
+ %arrayidx.i.6 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 24
+ %12 = load float, ptr addrspace(1) %arrayidx.i.6, align 4
+ %arrayidx3.i.6 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 24
+ %13 = load float, ptr addrspace(1) %arrayidx3.i.6, align 4
+ %mul.i.6 = fmul contract float %12, %13
+ %add.i.6 = fadd contract float %add.i.5, %mul.i.6
+ %arrayidx.i.7 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 28
+ %14 = load float, ptr addrspace(1) %arrayidx.i.7, align 4
+ %arrayidx3.i.7 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 28
+ %15 = load float, ptr addrspace(1) %arrayidx3.i.7, align 4
+ %mul.i.7 = fmul contract float %14, %15
+ %add.i.7 = fadd contract float %add.i.6, %mul.i.7
+ %arrayidx.i.8 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 32
+ %16 = load float, ptr addrspace(1) %arrayidx.i.8, align 4
+ %arrayidx3.i.8 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 32
+ %17 = load float, ptr addrspace(1) %arrayidx3.i.8, align 4
+ %mul.i.8 = fmul contract float %16, %17
+ %add.i.8 = fadd contract float %add.i.7, %mul.i.8
+ %arrayidx.i.9 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 36
+ %18 = load float, ptr addrspace(1) %arrayidx.i.9, align 4
+ %arrayidx3.i.9 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 36
+ %19 = load float, ptr addrspace(1) %arrayidx3.i.9, align 4
+ %mul.i.9 = fmul contract float %18, %19
+ %add.i.9 = fadd contract float %add.i.8, %mul.i.9
+ %arrayidx.i.10 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 40
+ %20 = load float, ptr addrspace(1) %arrayidx.i.10, align 4
+ %arrayidx3.i.10 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 40
+ %21 = load float, ptr addrspace(1) %arrayidx3.i.10, align 4
+ %mul.i.10 = fmul contract float %20, %21
+ %add.i.10 = fadd contract float %add.i.9, %mul.i.10
+ %arrayidx.i.11 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 44
+ %22 = load float, ptr addrspace(1) %arrayidx.i.11, align 4
+ %arrayidx3.i.11 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 44
+ %23 = load float, ptr addrspace(1) %arrayidx3.i.11, align 4
+ %mul.i.11 = fmul contract float %22, %23
+ %add.i.11 = fadd contract float %add.i.10, %mul.i.11
+ %arrayidx.i.12 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 48
+ %24 = load float, ptr addrspace(1) %arrayidx.i.12, align 4
+ %arrayidx3.i.12 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 48
+ %25 = load float, ptr addrspace(1) %arrayidx3.i.12, align 4
+ %mul.i.12 = fmul contract float %24, %25
+ %add.i.12 = fadd contract float %add.i.11, %mul.i.12
+ %arrayidx.i.13 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 52
+ %26 = load float, ptr addrspace(1) %arrayidx.i.13, align 4
+ %arrayidx3.i.13 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 52
+ %27 = load float, ptr addrspace(1) %arrayidx3.i.13, align 4
+ %mul.i.13 = fmul contract float %26, %27
+ %add.i.13 = fadd contract float %add.i.12, %mul.i.13
+ %arrayidx.i.14 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 56
+ %28 = load float, ptr addrspace(1) %arrayidx.i.14, align 4
+ %arrayidx3.i.14 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 56
+ %29 = load float, ptr addrspace(1) %arrayidx3.i.14, align 4
+ %mul.i.14 = fmul contract float %28, %29
+ %add.i.14 = fadd contract float %add.i.13, %mul.i.14
+ %arrayidx.i.15 = getelementptr inbounds nuw i8, ptr addrspace(1) %add.ptr.i, i64 60
+ %30 = load float, ptr addrspace(1) %arrayidx.i.15, align 4
+ %arrayidx3.i.15 = getelementptr inbounds nuw i8, ptr addrspace(1) %mask, i64 60
+ %31 = load float, ptr addrspace(1) %arrayidx3.i.15, align 4
+ %mul.i.15 = fmul contract float %30, %31
+ %add.i.15 = fadd contract float %add.i.14, %mul.i.15
+ %arrayidx5.i = getelementptr inbounds [4 x i8], ptr addrspace(1) %out, i64 %idx.ext.i
+ store float %add.i.15, ptr addrspace(1) %arrayidx5.i, align 4
+ ret void
+}
+
+declare i64 @_Z13get_global_idj(i32 noundef)
+
+; The narrow integer lanes below make the scalar reduction cheap enough for
+; the split window path to reconsider it. The window has to see the same
+; coalesced loads and stay scalar too.
+
+define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
+; CHECK-LABEL: define float @dot_i16_window(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[M0:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; CHECK-NEXT: [[M1:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
+; CHECK-NEXT: [[M2:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; CHECK-NEXT: [[M3:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; CHECK-NEXT: ret float [[S3]]
+;
+entry:
+ %a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
+ %a2 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 4
+ %a3 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 6
+ %b1 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 4
+ %b2 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 8
+ %b3 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 12
+ %i0 = load i16, ptr addrspace(1) %a, align 2
+ %i1 = load i16, ptr addrspace(1) %a1, align 2
+ %i2 = load i16, ptr addrspace(1) %a2, align 2
+ %i3 = load i16, ptr addrspace(1) %a3, align 2
+ %f0 = sitofp i16 %i0 to float
+ %f1 = sitofp i16 %i1 to float
+ %f2 = sitofp i16 %i2 to float
+ %f3 = sitofp i16 %i3 to float
+ %w0 = load float, ptr addrspace(1) %b, align 4
+ %w1 = load float, ptr addrspace(1) %b1, align 4
+ %w2 = load float, ptr addrspace(1) %b2, align 4
+ %w3 = load float, ptr addrspace(1) %b3, align 4
+ %m0 = fmul contract float %f0, %w0
+ %m1 = fmul contract float %f1, %w1
+ %m2 = fmul contract float %f2, %w2
+ %m3 = fmul contract float %f3, %w3
+ %s0 = fadd contract float %acc, %m0
+ %s1 = fadd contract float %s0, %m1
+ %s2 = fadd contract float %s1, %m2
+ %s3 = fadd contract float %s2, %m3
+ ret float %s3
+}
+
+; An ordered contract fadd reduction of eight products inside a loop of 16
+; iterations. The load saving that never materializes is cancelled in every
+; iteration, so the reduction stays scalar and fuses into fmas here too.
+define float @dot8_ordered_loop(ptr addrspace(1) noalias readonly align 4 %a0, ptr addrspace(1) noalias readonly align 4 %b0, float %acc0) {
+; CHECK-LABEL: define float @dot8_ordered_loop(
+; CHECK-SAME: ptr addrspace(1) noalias readonly align 4 [[A0:%.*]], ptr addrspace(1) noalias readonly align 4 [[B0:%.*]], float [[ACC0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi float [ [[ACC0]], %[[ENTRY]] ], [ [[R7:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[OFF:%.*]] = shl i64 [[I]], 3
+; CHECK-NEXT: [[A:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A0]], i64 [[OFF]]
+; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B0]], i64 [[OFF]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[M0:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[R0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; CHECK-NEXT: [[M1:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[R0]], [[M1]]
+; CHECK-NEXT: [[M2:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[R1]], [[M2]]
+; CHECK-NEXT: [[M3:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[R2]], [[M3]]
+; CHECK-NEXT: [[M4:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[R4:%.*]] = fadd contract float [[R3]], [[M4]]
+; CHECK-NEXT: [[M5:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[R5:%.*]] = fadd contract float [[R4]], [[M5]]
+; CHECK-NEXT: [[M6:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[R6:%.*]] = fadd contract float [[R5]], [[M6]]
+; CHECK-NEXT: [[M7:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[R7]] = fadd contract float [[R6]], [[M7]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], 16
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float [[R7]]
+;
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ %acc0, %entry ], [ %r7, %loop ]
+ %off = shl i64 %i, 3
+ %a = getelementptr inbounds float, ptr addrspace(1) %a0, i64 %off
+ %b = getelementptr inbounds float, ptr addrspace(1) %b0, i64 %off
+ %la0 = load float, ptr addrspace(1) %a, align 4
+ %lb0 = load float, ptr addrspace(1) %b, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %r0 = fadd contract float %acc, %m0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %r1 = fadd contract float %r0, %m1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %r2 = fadd contract float %r1, %m2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %r3 = fadd contract float %r2, %m3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %r4 = fadd contract float %r3, %m4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %r5 = fadd contract float %r4, %m5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %r6 = fadd contract float %r5, %m6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %r7 = fadd contract float %r6, %m7
+ %i.next = add nuw nsw i64 %i, 1
+ %c = icmp ult i64 %i.next, 16
+ br i1 %c, label %loop, label %exit
+exit:
+ ret float %r7
+}
+
+; An ordered contract fadd reduction of eight products over LDS with the
+; first float load of each pointer aligned to 16. The float loads coalesce
+; into two loads per pointer, and a bundle of them is judged by the best
+; alignment among its loads, so a bundle starting at the second float
+; coalesces as well. The reduction stays scalar and fuses into fmas.
+define float @dot8_ordered_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b, float %acc) {
+; CHECK-LABEL: define float @dot8_ordered_lds_a16(
+; CHECK-SAME: ptr addrspace(3) [[A:%.*]], ptr addrspace(3) [[B:%.*]], float [[ACC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(3) [[PA0]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(3) [[PB0]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP10]]
+; CHECK-NEXT: ret float [[S7]]
+;
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(3) %a, i32 0
+ %la0 = load float, ptr addrspace(3) %pa0, align 16
+ %pb0 = getelementptr inbounds float, ptr addrspace(3) %b, i32 0
+ %lb0 = load float, ptr addrspace(3) %pb0, align 16
+ %m0 = fmul contract float %la0, %lb0
+ %s0 = fadd contract float %acc, %m0
+ %pa1 = getelementptr inbounds float, ptr addrspace(3) %a, i32 1
+ %la1 = load float, ptr addrspace(3) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(3) %b, i32 1
+ %lb1 = load float, ptr addrspace(3) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %s1 = fadd contract float %s0, %m1
+ %pa2 = getelementptr inbounds float, ptr addrspace(3) %a, i32 2
+ %la2 = load float, ptr addrspace(3) %pa2, align 8
+ %pb2 = getelementptr inbounds float, ptr addrspace(3) %b, i32 2
+ %lb2 = load float, ptr addrspace(3) %pb2, align 8
+ %m2 = fmul contract float %la2, %lb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds float, ptr addrspace(3) %a, i32 3
+ %la3 = load float, ptr addrspace(3) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(3) %b, i32 3
+ %lb3 = load float, ptr addrspace(3) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds float, ptr addrspace(3) %a, i32 4
+ %la4 = load float, ptr addrspace(3) %pa4, align 16
+ %pb4 = getelementptr inbounds float, ptr addrspace(3) %b, i32 4
+ %lb4 = load float, ptr addrspace(3) %pb4, align 16
+ %m4 = fmul contract float %la4, %lb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds float, ptr addrspace(3) %a, i32 5
+ %la5 = load float, ptr addrspace(3) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(3) %b, i32 5
+ %lb5 = load float, ptr addrspace(3) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds float, ptr addrspace(3) %a, i32 6
+ %la6 = load float, ptr addrspace(3) %pa6, align 8
+ %pb6 = getelementptr inbounds float, ptr addrspace(3) %b, i32 6
+ %lb6 = load float, ptr addrspace(3) %pb6, align 8
+ %m6 = fmul contract float %la6, %lb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds float, ptr addrspace(3) %a, i32 7
+ %la7 = load float, ptr addrspace(3) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(3) %b, i32 7
+ %lb7 = load float, ptr addrspace(3) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+
+; An ordered contract fadd reduction of eight products that starts from the
+; first product instead of an accumulator. Seven products fuse into fmas and
+; the eighth stays an fmul. A vector fmul of the products would feed scalar
+; fadds through extracts and fuse with none of them, so it is priced as a
+; plain vector fmul and the reduction stays scalar. gfx1030 emits a vector
+; fmul lane by lane and every lane still fuses with its fadd, so it keeps the
+; vector form.
+define float @dot8_ordered_noacc(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b) {
+; CHECK-LABEL: define float @dot8_ordered_noacc(
+; CHECK-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[PA0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PB0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP10]]
+; CHECK-NEXT: ret float [[S7]]
+;
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %s1 = fadd contract float %m0, %m1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+
+; The same reduction over products of sign extended i16 loads. On targets
+; where converting the vector of i16 lanes costs more than the scalar
+; conversions, the loads and conversions are left scalar and only the
+; products are left to vectorize. A vector fmul of the products gives up the
+; fmas, so the reduction stays scalar. gfx1030 emits the vector fmul lane by
+; lane, fuses every lane and keeps the vector form.
+define float @dot8_sitofp_i16(ptr addrspace(1) noalias readonly align 16 %a, ptr addrspace(1) noalias readonly align 16 %b) {
+; GFX9-LABEL: define float @dot8_sitofp_i16(
+; GFX9-SAME: ptr addrspace(1) noalias readonly align 16 [[A:%.*]], ptr addrspace(1) noalias readonly align 16 [[B:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[PA0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 0
+; GFX9-NEXT: [[PB0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 0
+; GFX9-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr addrspace(1) [[PA0]], align 16
+; GFX9-NEXT: [[TMP1:%.*]] = sitofp <8 x i16> [[TMP0]] to <8 x float>
+; GFX9-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr addrspace(1) [[PB0]], align 16
+; GFX9-NEXT: [[TMP3:%.*]] = sitofp <8 x i16> [[TMP2]] to <8 x float>
+; GFX9-NEXT: [[TMP4:%.*]] = fmul contract <8 x float> [[TMP1]], [[TMP3]]
+; GFX9-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP4]], i64 0
+; GFX9-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP4]], i64 1
+; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[TMP5]], [[TMP6]]
+; GFX9-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP4]], i64 2
+; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP7]]
+; GFX9-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP4]], i64 3
+; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP8]]
+; GFX9-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP4]], i64 4
+; GFX9-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP9]]
+; GFX9-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP4]], i64 5
+; GFX9-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP10]]
+; GFX9-NEXT: [[TMP11:%.*]] = extractelement <8 x float> [[TMP4]], i64 6
+; GFX9-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP11]]
+; GFX9-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[TMP4]], i64 7
+; GFX9-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP12]]
+; GFX9-NEXT: ret float [[S7]]
+;
+; GFX1030-LABEL: define float @dot8_sitofp_i16(
+; GFX1030-SAME: ptr addrspace(1) noalias readonly align 16 [[A:%.*]], ptr addrspace(1) noalias readonly align 16 [[B:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[PA0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 0
+; GFX1030-NEXT: [[PB0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 0
+; GFX1030-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr addrspace(1) [[PA0]], align 16
+; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <8 x i16> [[TMP0]] to <8 x float>
+; GFX1030-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr addrspace(1) [[PB0]], align 16
+; GFX1030-NEXT: [[TMP3:%.*]] = sitofp <8 x i16> [[TMP2]] to <8 x float>
+; GFX1030-NEXT: [[TMP4:%.*]] = fmul contract <8 x float> [[TMP1]], [[TMP3]]
+; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP4]], i64 0
+; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP4]], i64 1
+; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[TMP5]], [[TMP6]]
+; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP4]], i64 2
+; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP7]]
+; GFX1030-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP4]], i64 3
+; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP8]]
+; GFX1030-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP4]], i64 4
+; GFX1030-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP9]]
+; GFX1030-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP4]], i64 5
+; GFX1030-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP10]]
+; GFX1030-NEXT: [[TMP11:%.*]] = extractelement <8 x float> [[TMP4]], i64 6
+; GFX1030-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP11]]
+; GFX1030-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[TMP4]], i64 7
+; GFX1030-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP12]]
+; GFX1030-NEXT: ret float [[S7]]
+;
+; GFX1250-LABEL: define float @dot8_sitofp_i16(
+; GFX1250-SAME: ptr addrspace(1) noalias readonly align 16 [[A:%.*]], ptr addrspace(1) noalias readonly align 16 [[B:%.*]]) {
+; GFX1250-NEXT: [[ENTRY:.*:]]
+; GFX1250-NEXT: [[PA0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 0
+; GFX1250-NEXT: [[LA0:%.*]] = load i16, ptr addrspace(1) [[PA0]], align 16
+; GFX1250-NEXT: [[FA0:%.*]] = sitofp i16 [[LA0]] to float
+; GFX1250-NEXT: [[PB0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 0
+; GFX1250-NEXT: [[LB0:%.*]] = load i16, ptr addrspace(1) [[PB0]], align 16
+; GFX1250-NEXT: [[FB0:%.*]] = sitofp i16 [[LB0]] to float
+; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[FA0]], [[FB0]]
+; GFX1250-NEXT: [[PA1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 1
+; GFX1250-NEXT: [[LA1:%.*]] = load i16, ptr addrspace(1) [[PA1]], align 2
+; GFX1250-NEXT: [[FA1:%.*]] = sitofp i16 [[LA1]] to float
+; GFX1250-NEXT: [[PB1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 1
+; GFX1250-NEXT: [[LB1:%.*]] = load i16, ptr addrspace(1) [[PB1]], align 2
+; GFX1250-NEXT: [[FB1:%.*]] = sitofp i16 [[LB1]] to float
+; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[FA1]], [[FB1]]
+; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[M0]], [[M1]]
+; GFX1250-NEXT: [[PA2:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 2
+; GFX1250-NEXT: [[LA2:%.*]] = load i16, ptr addrspace(1) [[PA2]], align 4
+; GFX1250-NEXT: [[FA2:%.*]] = sitofp i16 [[LA2]] to float
+; GFX1250-NEXT: [[PB2:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 2
+; GFX1250-NEXT: [[LB2:%.*]] = load i16, ptr addrspace(1) [[PB2]], align 4
+; GFX1250-NEXT: [[FB2:%.*]] = sitofp i16 [[LB2]] to float
+; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[FA2]], [[FB2]]
+; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX1250-NEXT: [[PA3:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 3
+; GFX1250-NEXT: [[LA3:%.*]] = load i16, ptr addrspace(1) [[PA3]], align 2
+; GFX1250-NEXT: [[FA3:%.*]] = sitofp i16 [[LA3]] to float
+; GFX1250-NEXT: [[PB3:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 3
+; GFX1250-NEXT: [[LB3:%.*]] = load i16, ptr addrspace(1) [[PB3]], align 2
+; GFX1250-NEXT: [[FB3:%.*]] = sitofp i16 [[LB3]] to float
+; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[FA3]], [[FB3]]
+; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX1250-NEXT: [[PA4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 4
+; GFX1250-NEXT: [[LA4:%.*]] = load i16, ptr addrspace(1) [[PA4]], align 8
+; GFX1250-NEXT: [[FA4:%.*]] = sitofp i16 [[LA4]] to float
+; GFX1250-NEXT: [[PB4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 4
+; GFX1250-NEXT: [[LB4:%.*]] = load i16, ptr addrspace(1) [[PB4]], align 8
+; GFX1250-NEXT: [[FB4:%.*]] = sitofp i16 [[LB4]] to float
+; GFX1250-NEXT: [[M4:%.*]] = fmul contract float [[FA4]], [[FB4]]
+; GFX1250-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[M4]]
+; GFX1250-NEXT: [[PA5:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 5
+; GFX1250-NEXT: [[LA5:%.*]] = load i16, ptr addrspace(1) [[PA5]], align 2
+; GFX1250-NEXT: [[FA5:%.*]] = sitofp i16 [[LA5]] to float
+; GFX1250-NEXT: [[PB5:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 5
+; GFX1250-NEXT: [[LB5:%.*]] = load i16, ptr addrspace(1) [[PB5]], align 2
+; GFX1250-NEXT: [[FB5:%.*]] = sitofp i16 [[LB5]] to float
+; GFX1250-NEXT: [[M5:%.*]] = fmul contract float [[FA5]], [[FB5]]
+; GFX1250-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[M5]]
+; GFX1250-NEXT: [[PA6:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 6
+; GFX1250-NEXT: [[LA6:%.*]] = load i16, ptr addrspace(1) [[PA6]], align 4
+; GFX1250-NEXT: [[FA6:%.*]] = sitofp i16 [[LA6]] to float
+; GFX1250-NEXT: [[PB6:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 6
+; GFX1250-NEXT: [[LB6:%.*]] = load i16, ptr addrspace(1) [[PB6]], align 4
+; GFX1250-NEXT: [[FB6:%.*]] = sitofp i16 [[LB6]] to float
+; GFX1250-NEXT: [[M6:%.*]] = fmul contract float [[FA6]], [[FB6]]
+; GFX1250-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[M6]]
+; GFX1250-NEXT: [[PA7:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 7
+; GFX1250-NEXT: [[LA7:%.*]] = load i16, ptr addrspace(1) [[PA7]], align 2
+; GFX1250-NEXT: [[FA7:%.*]] = sitofp i16 [[LA7]] to float
+; GFX1250-NEXT: [[PB7:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 7
+; GFX1250-NEXT: [[LB7:%.*]] = load i16, ptr addrspace(1) [[PB7]], align 2
+; GFX1250-NEXT: [[FB7:%.*]] = sitofp i16 [[LB7]] to float
+; GFX1250-NEXT: [[M7:%.*]] = fmul contract float [[FA7]], [[FB7]]
+; GFX1250-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[M7]]
+; GFX1250-NEXT: ret float [[S7]]
+;
+entry:
+ %pa0 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 0
+ %la0 = load i16, ptr addrspace(1) %pa0, align 16
+ %fa0 = sitofp i16 %la0 to float
+ %pb0 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 0
+ %lb0 = load i16, ptr addrspace(1) %pb0, align 16
+ %fb0 = sitofp i16 %lb0 to float
+ %m0 = fmul contract float %fa0, %fb0
+ %pa1 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 1
+ %la1 = load i16, ptr addrspace(1) %pa1, align 2
+ %fa1 = sitofp i16 %la1 to float
+ %pb1 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 1
+ %lb1 = load i16, ptr addrspace(1) %pb1, align 2
+ %fb1 = sitofp i16 %lb1 to float
+ %m1 = fmul contract float %fa1, %fb1
+ %s1 = fadd contract float %m0, %m1
+ %pa2 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 2
+ %la2 = load i16, ptr addrspace(1) %pa2, align 4
+ %fa2 = sitofp i16 %la2 to float
+ %pb2 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 2
+ %lb2 = load i16, ptr addrspace(1) %pb2, align 4
+ %fb2 = sitofp i16 %lb2 to float
+ %m2 = fmul contract float %fa2, %fb2
+ %s2 = fadd contract float %s1, %m2
+ %pa3 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 3
+ %la3 = load i16, ptr addrspace(1) %pa3, align 2
+ %fa3 = sitofp i16 %la3 to float
+ %pb3 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 3
+ %lb3 = load i16, ptr addrspace(1) %pb3, align 2
+ %fb3 = sitofp i16 %lb3 to float
+ %m3 = fmul contract float %fa3, %fb3
+ %s3 = fadd contract float %s2, %m3
+ %pa4 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 4
+ %la4 = load i16, ptr addrspace(1) %pa4, align 8
+ %fa4 = sitofp i16 %la4 to float
+ %pb4 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 4
+ %lb4 = load i16, ptr addrspace(1) %pb4, align 8
+ %fb4 = sitofp i16 %lb4 to float
+ %m4 = fmul contract float %fa4, %fb4
+ %s4 = fadd contract float %s3, %m4
+ %pa5 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 5
+ %la5 = load i16, ptr addrspace(1) %pa5, align 2
+ %fa5 = sitofp i16 %la5 to float
+ %pb5 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 5
+ %lb5 = load i16, ptr addrspace(1) %pb5, align 2
+ %fb5 = sitofp i16 %lb5 to float
+ %m5 = fmul contract float %fa5, %fb5
+ %s5 = fadd contract float %s4, %m5
+ %pa6 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 6
+ %la6 = load i16, ptr addrspace(1) %pa6, align 4
+ %fa6 = sitofp i16 %la6 to float
+ %pb6 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 6
+ %lb6 = load i16, ptr addrspace(1) %pb6, align 4
+ %fb6 = sitofp i16 %lb6 to float
+ %m6 = fmul contract float %fa6, %fb6
+ %s6 = fadd contract float %s5, %m6
+ %pa7 = getelementptr inbounds i16, ptr addrspace(1) %a, i64 7
+ %la7 = load i16, ptr addrspace(1) %pa7, align 2
+ %fa7 = sitofp i16 %la7 to float
+ %pb7 = getelementptr inbounds i16, ptr addrspace(1) %b, i64 7
+ %lb7 = load i16, ptr addrspace(1) %pb7, align 2
+ %fb7 = sitofp i16 %lb7 to float
+ %m7 = fmul contract float %fa7, %fb7
+ %s7 = fadd contract float %s6, %m7
+ ret float %s7
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX90A: {{.*}}
+; GFX942: {{.*}}
+; GFX942-950: {{.*}}
+; GFX950: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reassoc-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reassoc-reduction-coalesced-loads.ll
new file mode 100644
index 0000000000000..96bde4599ddbe
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reassoc-reduction-coalesced-loads.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=PACKED,GFX90A
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s --check-prefixes=PACKED,GFX942
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=PACKED,GFX950
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefix=SCALARFMA
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=PACKED,GFX1250
+
+; Reassociable fadd reductions over consecutive global loads keep their load
+; saving although the scalar loads coalesce. The vector fmuls of dot8 fuse
+; into the reduction, so no fma is lost, and sum8 has no fma at stake.
+
+define float @dot8(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b) {
+; PACKED-LABEL: define float @dot8(
+; PACKED-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; PACKED-NEXT: [[ENTRY:.*:]]
+; PACKED-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; PACKED-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; PACKED-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[PA0]], align 4
+; PACKED-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PB0]], align 4
+; PACKED-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
+; PACKED-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP2]])
+; PACKED-NEXT: ret float [[TMP6]]
+;
+; SCALARFMA-LABEL: define float @dot8(
+; SCALARFMA-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; SCALARFMA-NEXT: [[ENTRY:.*:]]
+; SCALARFMA-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; SCALARFMA-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; SCALARFMA-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[PA0]], align 4
+; SCALARFMA-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[PB0]], align 4
+; SCALARFMA-NEXT: [[TMP2:%.*]] = fmul contract <4 x float> [[TMP0]], [[TMP1]]
+; SCALARFMA-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 4
+; SCALARFMA-NEXT: [[PB4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 4
+; SCALARFMA-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr addrspace(1) [[PA4]], align 4
+; SCALARFMA-NEXT: [[TMP4:%.*]] = load <4 x float>, ptr addrspace(1) [[PB4]], align 4
+; SCALARFMA-NEXT: [[TMP5:%.*]] = fmul contract <4 x float> [[TMP3]], [[TMP4]]
+; SCALARFMA-NEXT: [[RDX_OP:%.*]] = fadd fast <4 x float> [[TMP2]], [[TMP5]]
+; SCALARFMA-NEXT: [[S7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[RDX_OP]])
+; SCALARFMA-NEXT: ret float [[S7]]
+;
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pb0 = getelementptr inbounds float, ptr addrspace(1) %b, i64 0
+ %lb0 = load float, ptr addrspace(1) %pb0, align 4
+ %m0 = fmul contract float %la0, %lb0
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pb1 = getelementptr inbounds float, ptr addrspace(1) %b, i64 1
+ %lb1 = load float, ptr addrspace(1) %pb1, align 4
+ %m1 = fmul contract float %la1, %lb1
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pb2 = getelementptr inbounds float, ptr addrspace(1) %b, i64 2
+ %lb2 = load float, ptr addrspace(1) %pb2, align 4
+ %m2 = fmul contract float %la2, %lb2
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pb3 = getelementptr inbounds float, ptr addrspace(1) %b, i64 3
+ %lb3 = load float, ptr addrspace(1) %pb3, align 4
+ %m3 = fmul contract float %la3, %lb3
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pb4 = getelementptr inbounds float, ptr addrspace(1) %b, i64 4
+ %lb4 = load float, ptr addrspace(1) %pb4, align 4
+ %m4 = fmul contract float %la4, %lb4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pb5 = getelementptr inbounds float, ptr addrspace(1) %b, i64 5
+ %lb5 = load float, ptr addrspace(1) %pb5, align 4
+ %m5 = fmul contract float %la5, %lb5
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pb6 = getelementptr inbounds float, ptr addrspace(1) %b, i64 6
+ %lb6 = load float, ptr addrspace(1) %pb6, align 4
+ %m6 = fmul contract float %la6, %lb6
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %pb7 = getelementptr inbounds float, ptr addrspace(1) %b, i64 7
+ %lb7 = load float, ptr addrspace(1) %pb7, align 4
+ %m7 = fmul contract float %la7, %lb7
+ %s1 = fadd fast float %m0, %m1
+ %s2 = fadd fast float %s1, %m2
+ %s3 = fadd fast float %s2, %m3
+ %s4 = fadd fast float %s3, %m4
+ %s5 = fadd fast float %s4, %m5
+ %s6 = fadd fast float %s5, %m6
+ %s7 = fadd fast float %s6, %m7
+ ret float %s7
+}
+
+define float @sum8(ptr addrspace(1) noalias readonly align 4 %a) {
+; PACKED-LABEL: define float @sum8(
+; PACKED-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]]) {
+; PACKED-NEXT: [[ENTRY:.*:]]
+; PACKED-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; PACKED-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[PA0]], align 4
+; PACKED-NEXT: [[TMP1:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
+; PACKED-NEXT: ret float [[TMP1]]
+;
+; SCALARFMA-LABEL: define float @sum8(
+; SCALARFMA-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]]) {
+; SCALARFMA-NEXT: [[ENTRY:.*:]]
+; SCALARFMA-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; SCALARFMA-NEXT: [[LA0:%.*]] = load float, ptr addrspace(1) [[PA0]], align 4
+; SCALARFMA-NEXT: [[PA1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 1
+; SCALARFMA-NEXT: [[LA1:%.*]] = load float, ptr addrspace(1) [[PA1]], align 4
+; SCALARFMA-NEXT: [[PA2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 2
+; SCALARFMA-NEXT: [[LA2:%.*]] = load float, ptr addrspace(1) [[PA2]], align 4
+; SCALARFMA-NEXT: [[PA3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 3
+; SCALARFMA-NEXT: [[LA3:%.*]] = load float, ptr addrspace(1) [[PA3]], align 4
+; SCALARFMA-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 4
+; SCALARFMA-NEXT: [[LA4:%.*]] = load float, ptr addrspace(1) [[PA4]], align 4
+; SCALARFMA-NEXT: [[PA5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 5
+; SCALARFMA-NEXT: [[LA5:%.*]] = load float, ptr addrspace(1) [[PA5]], align 4
+; SCALARFMA-NEXT: [[PA6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 6
+; SCALARFMA-NEXT: [[LA6:%.*]] = load float, ptr addrspace(1) [[PA6]], align 4
+; SCALARFMA-NEXT: [[PA7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 7
+; SCALARFMA-NEXT: [[LA7:%.*]] = load float, ptr addrspace(1) [[PA7]], align 4
+; SCALARFMA-NEXT: [[S1:%.*]] = fadd fast float [[LA0]], [[LA1]]
+; SCALARFMA-NEXT: [[S2:%.*]] = fadd fast float [[S1]], [[LA2]]
+; SCALARFMA-NEXT: [[S3:%.*]] = fadd fast float [[S2]], [[LA3]]
+; SCALARFMA-NEXT: [[S4:%.*]] = fadd fast float [[S3]], [[LA4]]
+; SCALARFMA-NEXT: [[S5:%.*]] = fadd fast float [[S4]], [[LA5]]
+; SCALARFMA-NEXT: [[S6:%.*]] = fadd fast float [[S5]], [[LA6]]
+; SCALARFMA-NEXT: [[S7:%.*]] = fadd fast float [[S6]], [[LA7]]
+; SCALARFMA-NEXT: ret float [[S7]]
+;
+entry:
+ %pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
+ %la0 = load float, ptr addrspace(1) %pa0, align 4
+ %pa1 = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
+ %la1 = load float, ptr addrspace(1) %pa1, align 4
+ %pa2 = getelementptr inbounds float, ptr addrspace(1) %a, i64 2
+ %la2 = load float, ptr addrspace(1) %pa2, align 4
+ %pa3 = getelementptr inbounds float, ptr addrspace(1) %a, i64 3
+ %la3 = load float, ptr addrspace(1) %pa3, align 4
+ %pa4 = getelementptr inbounds float, ptr addrspace(1) %a, i64 4
+ %la4 = load float, ptr addrspace(1) %pa4, align 4
+ %pa5 = getelementptr inbounds float, ptr addrspace(1) %a, i64 5
+ %la5 = load float, ptr addrspace(1) %pa5, align 4
+ %pa6 = getelementptr inbounds float, ptr addrspace(1) %a, i64 6
+ %la6 = load float, ptr addrspace(1) %pa6, align 4
+ %pa7 = getelementptr inbounds float, ptr addrspace(1) %a, i64 7
+ %la7 = load float, ptr addrspace(1) %pa7, align 4
+ %s1 = fadd fast float %la0, %la1
+ %s2 = fadd fast float %s1, %la2
+ %s3 = fadd fast float %s2, %la3
+ %s4 = fadd fast float %s3, %la4
+ %s5 = fadd fast float %s4, %la5
+ %s6 = fadd fast float %s5, %la6
+ %s7 = fadd fast float %s6, %la7
+ ret float %s7
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1250: {{.*}}
+; GFX90A: {{.*}}
+; GFX942: {{.*}}
+; GFX950: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/udiv-strictfp.ll b/llvm/test/Transforms/SLPVectorizer/X86/udiv-strictfp.ll
new file mode 100644
index 0000000000000..24ac6a07ab2ff
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/udiv-strictfp.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell -passes=slp-vectorizer -S | FileCheck %s
+
+; Four divisions by variable divisors, added up. The target prices a vector
+; division by variable divisors as a float divide sequence, which needs the
+; 512-bit rounding control under strictfp. The function attribute reaches the
+; cost through the scalar division, so the strictfp copy stays scalar.
+define i32 @udiv4_sum(ptr noalias readonly align 4 %a, ptr noalias readonly align 4 %b) {
+; CHECK-LABEL: define i32 @udiv4_sum(
+; CHECK-SAME: ptr noalias readonly align 4 [[A:%.*]], ptr noalias readonly align 4 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 0
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[PA0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[PB0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = udiv <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[PA2]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i32>, ptr [[PB2]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = udiv <2 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP2]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
+; CHECK-NEXT: [[S2:%.*]] = add i32 [[S1]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
+; CHECK-NEXT: [[S3:%.*]] = add i32 [[S2]], [[TMP9]]
+; CHECK-NEXT: ret i32 [[S3]]
+;
+entry:
+ %pa0 = getelementptr inbounds i32, ptr %a, i64 0
+ %la0 = load i32, ptr %pa0, align 4
+ %pb0 = getelementptr inbounds i32, ptr %b, i64 0
+ %lb0 = load i32, ptr %pb0, align 4
+ %d0 = udiv i32 %la0, %lb0
+ %pa1 = getelementptr inbounds i32, ptr %a, i64 1
+ %la1 = load i32, ptr %pa1, align 4
+ %pb1 = getelementptr inbounds i32, ptr %b, i64 1
+ %lb1 = load i32, ptr %pb1, align 4
+ %d1 = udiv i32 %la1, %lb1
+ %pa2 = getelementptr inbounds i32, ptr %a, i64 2
+ %la2 = load i32, ptr %pa2, align 4
+ %pb2 = getelementptr inbounds i32, ptr %b, i64 2
+ %lb2 = load i32, ptr %pb2, align 4
+ %d2 = udiv i32 %la2, %lb2
+ %pa3 = getelementptr inbounds i32, ptr %a, i64 3
+ %la3 = load i32, ptr %pa3, align 4
+ %pb3 = getelementptr inbounds i32, ptr %b, i64 3
+ %lb3 = load i32, ptr %pb3, align 4
+ %d3 = udiv i32 %la3, %lb3
+ %s1 = add i32 %d0, %d1
+ %s2 = add i32 %s1, %d2
+ %s3 = add i32 %s2, %d3
+ ret i32 %s3
+}
+
+define i32 @udiv4_sum_strictfp(ptr noalias readonly align 4 %a, ptr noalias readonly align 4 %b) strictfp {
+; CHECK-LABEL: define i32 @udiv4_sum_strictfp(
+; CHECK-SAME: ptr noalias readonly align 4 [[A:%.*]], ptr noalias readonly align 4 [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 0
+; CHECK-NEXT: [[LA0:%.*]] = load i32, ptr [[PA0]], align 4
+; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 0
+; CHECK-NEXT: [[LB0:%.*]] = load i32, ptr [[PB0]], align 4
+; CHECK-NEXT: [[D0:%.*]] = udiv i32 [[LA0]], [[LB0]]
+; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
+; CHECK-NEXT: [[LA1:%.*]] = load i32, ptr [[PA1]], align 4
+; CHECK-NEXT: [[PB1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 1
+; CHECK-NEXT: [[LB1:%.*]] = load i32, ptr [[PB1]], align 4
+; CHECK-NEXT: [[D1:%.*]] = udiv i32 [[LA1]], [[LB1]]
+; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[LA2:%.*]] = load i32, ptr [[PA2]], align 4
+; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: [[LB2:%.*]] = load i32, ptr [[PB2]], align 4
+; CHECK-NEXT: [[D2:%.*]] = udiv i32 [[LA2]], [[LB2]]
+; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 3
+; CHECK-NEXT: [[LA3:%.*]] = load i32, ptr [[PA3]], align 4
+; CHECK-NEXT: [[PB3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 3
+; CHECK-NEXT: [[LB3:%.*]] = load i32, ptr [[PB3]], align 4
+; CHECK-NEXT: [[D3:%.*]] = udiv i32 [[LA3]], [[LB3]]
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[D0]], [[D1]]
+; CHECK-NEXT: [[S2:%.*]] = add i32 [[S1]], [[D2]]
+; CHECK-NEXT: [[S3:%.*]] = add i32 [[S2]], [[D3]]
+; CHECK-NEXT: ret i32 [[S3]]
+;
+entry:
+ %pa0 = getelementptr inbounds i32, ptr %a, i64 0
+ %la0 = load i32, ptr %pa0, align 4
+ %pb0 = getelementptr inbounds i32, ptr %b, i64 0
+ %lb0 = load i32, ptr %pb0, align 4
+ %d0 = udiv i32 %la0, %lb0
+ %pa1 = getelementptr inbounds i32, ptr %a, i64 1
+ %la1 = load i32, ptr %pa1, align 4
+ %pb1 = getelementptr inbounds i32, ptr %b, i64 1
+ %lb1 = load i32, ptr %pb1, align 4
+ %d1 = udiv i32 %la1, %lb1
+ %pa2 = getelementptr inbounds i32, ptr %a, i64 2
+ %la2 = load i32, ptr %pa2, align 4
+ %pb2 = getelementptr inbounds i32, ptr %b, i64 2
+ %lb2 = load i32, ptr %pb2, align 4
+ %d2 = udiv i32 %la2, %lb2
+ %pa3 = getelementptr inbounds i32, ptr %a, i64 3
+ %la3 = load i32, ptr %pa3, align 4
+ %pb3 = getelementptr inbounds i32, ptr %b, i64 3
+ %lb3 = load i32, ptr %pb3, align 4
+ %d3 = udiv i32 %la3, %lb3
+ %s1 = add i32 %d0, %d1
+ %s2 = add i32 %s1, %d2
+ %s3 = add i32 %s2, %d3
+ ret i32 %s3
+}
More information about the llvm-commits
mailing list