[llvm] [AMDGPU][True16] Upstream more true16-related testing changes (NFC) (PR #208320)
Domenic Nutile via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 08:47:36 PDT 2026
https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/208320
>From 65759d08cb408268aefc2098a25e1b338b3270b3 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Wed, 8 Jul 2026 15:31:35 -0400
Subject: [PATCH] [AMDGPU][True16] Upstream more true16-related testing changes
---
.../GlobalISel/llvm.amdgcn.ballot.i32.ll | 33 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll | 4 +-
.../llvm.amdgcn.image.atomic.dim.a16.ll | 3211 +++++++++++++----
.../llvm.amdgcn.image.gather4.a16.dim.ll | 1461 ++++++--
.../llvm.amdgcn.image.load.2darraymsaa.a16.ll | 471 ++-
.../llvm.amdgcn.image.load.3d.a16.ll | 427 ++-
.../llvm.amdgcn.image.sample.g16.ll | 318 +-
.../llvm.amdgcn.raw.buffer.load.tfe.ll | 719 ++--
.../llvm.amdgcn.struct.buffer.load.tfe.ll | 760 ++--
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 139 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 103 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll | 132 +-
...wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 127 +-
...wmma-gfx12-w64-f16-f32-matrix-modifiers.ll | 85 +-
llvm/test/CodeGen/AMDGPU/fadd.f16.ll | 121 -
.../AMDGPU/gfx11-twoaddr-fma-fake16.mir | 102 +
.../test/CodeGen/AMDGPU/gfx11-twoaddr-fma.mir | 103 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll | 95 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 6 +-
.../AMDGPU/llvm.amdgcn.raw.buffer.load.ll | 398 +-
llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll | 80 +-
llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll | 212 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 2034 +++++++----
llvm/test/CodeGen/AMDGPU/mul.ll | 6 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 150 +-
llvm/test/CodeGen/AMDGPU/rotl.ll | 317 +-
llvm/test/CodeGen/AMDGPU/rotr.ll | 508 ++-
...schedule-regpressure-ilp-metric-spills.mir | 384 +-
llvm/test/CodeGen/AMDGPU/shrink-mad-fma.mir | 97 +
llvm/test/CodeGen/AMDGPU/shrink-true16.mir | 34 +-
llvm/test/CodeGen/AMDGPU/shrink16.mir | 53 +
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 106 -
.../CodeGen/AMDGPU/subreg16-allocator.mir | 85 +
.../AMDGPU/true16-copy-vgpr16-to-sgpr32.mir | 118 +
.../CodeGen/AMDGPU/true16_v2f16_reg_seq.ll | 171 +
llvm/test/CodeGen/AMDGPU/v_swap_b16.mir | 409 +++
.../AMDGPU/valu-mask-write-hazard-fake16.mir | 20 +
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 735 ++--
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 2625 ++++++++++++++
llvm/test/CodeGen/AMDGPU/waitcnt-vinterp.mir | 48 +-
...wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 39 +-
43 files changed, 13011 insertions(+), 4047 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma-fake16.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/shrink16.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/subreg16-allocator.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/true16-copy-vgpr16-to-sgpr32.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/true16_v2f16_reg_seq.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/v_swap_b16.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard-fake16.mir
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index 048f5710c880c..b10dda15ace2d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu10.10 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX11 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX11,GFX11-FAKE16 %s
declare i32 @llvm.amdgcn.ballot.i32(i1)
declare i32 @llvm.ctpop.i32(i32)
@@ -460,6 +461,33 @@ define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid
; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
+; PREGFX11-LABEL: non_cst_non_compare_input:
+; PREGFX11: ; %bb.0: ; %entry
+; PREGFX11-NEXT: s_and_b32 s0, s0, 1
+; PREGFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; PREGFX11-NEXT: s_cmp_lg_u32 s0, 0
+; PREGFX11-NEXT: s_cselect_b32 s0, exec_lo, 0
+; PREGFX11-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; PREGFX11-NEXT: s_xor_b32 s1, exec_lo, s1
+; PREGFX11-NEXT: ; %bb.1: ; %B
+; PREGFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 2, v2
+; PREGFX11-NEXT: s_andn2_b32 s0, s0, exec_lo
+; PREGFX11-NEXT: ; implicit-def: $vgpr2
+; PREGFX11-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; PREGFX11-NEXT: s_or_b32 s0, s0, s2
+; PREGFX11-NEXT: ; %bb.2: ; %Flow
+; PREGFX11-NEXT: s_andn2_saveexec_b32 s1, s1
+; PREGFX11-NEXT: ; %bb.3: ; %A
+; PREGFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 1, v2
+; PREGFX11-NEXT: s_andn2_b32 s0, s0, exec_lo
+; PREGFX11-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; PREGFX11-NEXT: s_or_b32 s0, s0, s2
+; PREGFX11-NEXT: ; %bb.4: ; %exit
+; PREGFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; PREGFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; PREGFX11-NEXT: v_mov_b32_e32 v2, s0
+; PREGFX11-NEXT: global_store_dword v[0:1], v2, off
+; PREGFX11-NEXT: s_endpgm
entry:
%cmp = icmp eq i32 %cond, 0
br i1 %cmp, label %A, label %B
@@ -478,3 +506,6 @@ exit:
store i32 %ballot, ptr addrspace(1) %out
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
index 83b739b5046d9..4de9c37d8a179 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
@@ -3,8 +3,8 @@
; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck --check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck --check-prefix=GFX10_W32 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -mattr=+wavefrontsize64 < %s | FileCheck --check-prefix=GFX10_W64 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11_W32 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GFX11_W64 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11_W32 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GFX11_W64 %s
define float @v_div_fmas_f32(float %a, float %b, float %c, i1 %d) {
; GFX7-LABEL: v_div_fmas_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index 89f73ba07d58d..fce5fd01065cc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -1,8 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
; GFX9-LABEL: atomic_swap_i32_1d:
@@ -37,21 +39,65 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_swap_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_swap_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_swap_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_swap_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_swap_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -91,21 +137,65 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -145,21 +235,65 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_sub_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_sub_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_sub_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_sub_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_sub_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_sub_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_sub_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_sub_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -199,21 +333,65 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_smin_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_min_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_smin_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_smin_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_smin_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_min_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_smin_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_min_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -253,21 +431,65 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_umin_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_min_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_umin_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_umin_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_umin_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_min_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_umin_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_min_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -307,21 +529,65 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_smax_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_max_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_smax_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_smax_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_smax_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_max_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_smax_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_max_int v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -361,21 +627,65 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_umax_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_max_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_umax_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_umax_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_umax_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_max_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_umax_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_max_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -415,21 +725,65 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_and_i321d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_and_i321d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_and_i321d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_and_i321d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_and_i321d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -469,21 +823,65 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_or_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_or_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_or_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_or_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_or_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -523,21 +921,65 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_xor_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_xor_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_xor_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_xor_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_xor_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -577,21 +1019,65 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_inc_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_inc_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_inc_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_inc_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_inc_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_inc_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_inc_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_inc_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -631,21 +1117,65 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_dec_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_dec_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_dec_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_dec_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_dec_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_dec_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_dec_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_dec_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -685,21 +1215,65 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_cmpswap_i32_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_cmpswap_i32_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_cmpswap_i32_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_cmpswap_i32_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_cmpswap_i32_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i16(i32 %cmp, i32 %swap, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -739,21 +1313,67 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i16(i32 %data, i16 %s, i16 %t, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -797,23 +1417,71 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_3d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_3d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_3d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_3d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_3d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.3d.i32.i16(i32 %data, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -857,23 +1525,71 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_cube:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_cube:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_cube:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_cube:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_cube:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.cube.i32.i16(i32 %data, i16 %s, i16 %t, i16 %face, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -913,21 +1629,67 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_1darray:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_1darray:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_1darray:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_1darray:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_1darray:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i16(i32 %data, i16 %s, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -971,23 +1733,71 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_2darray:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_2darray:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_2darray:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_2darray:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_2darray:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i16(i32 %data, i16 %s, i16 %t, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -1031,23 +1841,71 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_2dmsaa:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_2dmsaa:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_2dmsaa:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_2dmsaa:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_2dmsaa:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i16(i32 %data, i16 %s, i16 %t, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -1091,23 +1949,74 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_2darraymsaa:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_2darraymsaa:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[5:6], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_2darraymsaa:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_2darraymsaa:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_2darraymsaa:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i16(i32 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i32 %v to float
@@ -1147,21 +2056,65 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i32_1d_slc:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i32_1d_slc:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i32_1d_slc:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i32_1d_slc:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i32_1d_slc:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 2)
%out = bitcast i32 %v to float
@@ -1201,21 +2154,65 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_swap_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_swap_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_swap_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_swap_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_swap_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1255,21 +2252,65 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1309,21 +2350,65 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_sub_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_sub_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_sub_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_sub_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_sub_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_sub_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_sub_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_sub_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.sub.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1363,21 +2448,65 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_smin_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_min_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_smin_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_smin_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_smin_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_min_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_smin_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_min_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.smin.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1417,21 +2546,65 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_umin_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_min_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_umin_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_umin_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_umin_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_min_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_umin_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_min_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.umin.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1471,21 +2644,65 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_smax_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_max_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_smax_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_smax_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_smax_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_max_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_smax_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_max_int v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.smax.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1525,21 +2742,65 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_umax_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_max_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_umax_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_umax_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_umax_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_max_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_umax_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_max_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.umax.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1579,21 +2840,65 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_and_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_and_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_and_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_and_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_and_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.and.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1633,21 +2938,65 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_or_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_or_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_or_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_or_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_or_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.or.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1687,21 +3036,65 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_xor_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_xor_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_xor_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_xor_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_xor_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.xor.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1741,21 +3134,65 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_inc_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_inc_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_inc_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_inc_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_inc_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_inc_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_inc_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_inc_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.inc.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1795,21 +3232,65 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_dec_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_dec_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_dec_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_dec_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_dec_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_dec_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_dec_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_dec_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.dec.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1849,21 +3330,65 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_cmpswap_i64_1d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v4, s0, 16, v4
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_cmpswap_i64_1d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_cmpswap_i64_1d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, s0, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_cmpswap_i64_1d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_cmpswap_i64_1d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v4, s0, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i16(i64 %cmp, i64 %swap, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1903,21 +3428,67 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.2d.i64.i16(i64 %data, i16 %s, i16 %t, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -1961,23 +3532,71 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_3d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_3d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_3d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_3d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_3d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.3d.i64.i16(i64 %data, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2021,23 +3640,71 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_cube:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_cube:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_cube:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_cube:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_cube:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.cube.i64.i16(i64 %data, i16 %s, i16 %t, i16 %face , <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2077,21 +3744,67 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_1darray:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_1darray:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_1darray:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_1darray:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_1darray:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.1darray.i64.i16(i64 %data, i16 %s, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2135,23 +3848,71 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_2darray:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_2darray:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_2darray:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_2darray:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_2darray:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.2darray.i64.i16(i64 %data, i16 %s, i16 %t, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2195,23 +3956,71 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_2dmsaa:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_2dmsaa:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_2dmsaa:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_2dmsaa:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_2dmsaa:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i16(i64 %data, i16 %s, i16 %t, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2255,23 +4064,74 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_2darraymsaa:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX12-NEXT: v_lshl_or_b32 v3, v5, 16, v4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_2darraymsaa:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[6:7], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_2darraymsaa:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_2darraymsaa:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_2darraymsaa:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v3, v5, 16, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], [v2, v3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY th:TH_ATOMIC_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i16(i64 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
%out = bitcast i64 %v to <2 x float>
@@ -2311,21 +4171,65 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: atomic_add_i64_1d_slc:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: atomic_add_i64_1d_slc:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: atomic_add_i64_1d_slc:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: atomic_add_i64_1d_slc:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: atomic_add_i64_1d_slc:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_atomic_add_uint v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 2)
%out = bitcast i64 %v to <2 x float>
@@ -2375,3 +4279,6 @@ declare i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i16(i64, i16, i16, i16, <8
declare i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i16(i64, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index cb52ce10c30bf..a7843033a4081 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -1,8 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX10NSA %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX10NSA %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
; GFX9-LABEL: gather4_2d:
@@ -50,27 +52,93 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -122,27 +190,93 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_cube:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_cube:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_cube:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_cube:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_cube:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.cube.v4f32.f16(i32 1, half %s, half %t, half %face, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -194,27 +328,93 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_2darray:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_2darray:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_2darray:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_2darray:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_2darray:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.2darray.v4f32.f16(i32 1, half %s, half %t, half %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -266,27 +466,93 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_c v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_c v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_c v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -338,27 +604,93 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_cl v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f16(i32 1, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -412,27 +744,93 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -484,27 +882,93 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_b_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_b v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_b_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_b_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_b v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_b_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_b v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_b_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_b v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f16.f16(i32 1, half %bias, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -556,27 +1020,93 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_b_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_b_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_b_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_c_b v[0:3], v[0:2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_b_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_b_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f16.f16(i32 1, half %bias, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -630,27 +1160,93 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_b_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_b_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_b_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_b_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_b_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.2d.v4f32.f16.f16(i32 1, half %bias, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -705,27 +1301,93 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_b_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_b_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_b_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_b_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_b_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.2d.v4f32.f16.f16(i32 1, half %bias, float %zcompare, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -771,24 +1433,81 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_l_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_l v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_l_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_l_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_l_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_l_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f16(i32 1, half %s, half %t, half %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -836,24 +1555,81 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_l_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_l_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_l_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_l_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_l_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, half %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -899,24 +1675,81 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_lz_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_lz_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_lz_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_lz_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_lz_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -962,24 +1795,81 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: gather4_c_lz_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_c_lz v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: gather4_c_lz_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX11-TRUE16-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: gather4_c_lz_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX11-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX11-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX11-FAKE16-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: gather4_c_lz_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: s_mov_b32 s8, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
+; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
+; GFX12-TRUE16-NEXT: image_gather4_c_lz v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: gather4_c_lz_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_mov_b32 s15, s13
+; GFX12-FAKE16-NEXT: s_mov_b32 s14, s12
+; GFX12-FAKE16-NEXT: s_mov_b32 s13, s11
+; GFX12-FAKE16-NEXT: s_mov_b32 s12, s10
+; GFX12-FAKE16-NEXT: s_mov_b32 s11, s9
+; GFX12-FAKE16-NEXT: s_mov_b32 s10, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s9, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s8, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s2
+; GFX12-FAKE16-NEXT: image_gather4_c_lz v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
ret <4 x float> %v
@@ -1001,3 +1891,6 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 immarg, floa
declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f16(i32 immarg, float, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 4d1c0341c782c..26a7a9106b3b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -1,8 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw:
@@ -23,41 +25,93 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: load_2darraymsaa_v4f32_xyzw:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10PLUS-NEXT: s_mov_b32 s0, s2
-; GFX10PLUS-NEXT: s_mov_b32 s1, s3
-; GFX10PLUS-NEXT: s_mov_b32 s2, s4
-; GFX10PLUS-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX10PLUS-NEXT: s_mov_b32 s3, s5
-; GFX10PLUS-NEXT: s_mov_b32 s4, s6
-; GFX10PLUS-NEXT: s_mov_b32 s5, s7
-; GFX10PLUS-NEXT: s_mov_b32 s6, s8
-; GFX10PLUS-NEXT: s_mov_b32 s7, s9
-; GFX10PLUS-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_load v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_2darraymsaa_v4f32_xyzw:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_load v[0:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_load v[0:3], [v0, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_load v[0:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
ret <4 x float> %v
}
@@ -122,63 +176,121 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX10-NEXT: global_store_dword v5, v4, s[10:11]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: v_mov_b32_e32 v8, v5
-; GFX11-NEXT: v_mov_b32_e32 v9, v5
-; GFX11-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX11-NEXT: v_lshl_or_b32 v11, v3, 16, v2
-; GFX11-NEXT: s_mov_b32 s2, s4
-; GFX11-NEXT: s_mov_b32 s3, s5
-; GFX11-NEXT: s_mov_b32 s4, s6
-; GFX11-NEXT: s_mov_b32 s5, s7
-; GFX11-NEXT: s_mov_b32 s6, s8
-; GFX11-NEXT: s_mov_b32 s7, s9
-; GFX11-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-NEXT: v_mov_b32_e32 v1, v6
-; GFX11-NEXT: v_mov_b32_e32 v2, v7
-; GFX11-NEXT: v_mov_b32_e32 v3, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v10
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX11-TRUE16-NEXT: image_load v[0:4], v[5:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v9
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v10
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX12-TRUE16-NEXT: image_load v[0:4], [v5, v6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: v_mov_b32_e32 v7, v5
-; GFX12-NEXT: v_mov_b32_e32 v8, v5
-; GFX12-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v11, v3, 16, v2
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-NEXT: v_mov_b32_e32 v1, v6
-; GFX12-NEXT: v_mov_b32_e32 v2, v7
-; GFX12-NEXT: v_mov_b32_e32 v3, v8
-; GFX12-NEXT: v_mov_b32_e32 v4, v9
-; GFX12-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX12-FAKE16-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue { <4 x float>, i32 } %v, 0
%v.err = extractvalue { <4 x float>, i32 } %v, 1
@@ -246,63 +358,121 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX10-NEXT: global_store_dword v5, v4, s[10:11]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: v_mov_b32_e32 v8, v5
-; GFX11-NEXT: v_mov_b32_e32 v9, v5
-; GFX11-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX11-NEXT: v_lshl_or_b32 v11, v3, 16, v2
-; GFX11-NEXT: s_mov_b32 s2, s4
-; GFX11-NEXT: s_mov_b32 s3, s5
-; GFX11-NEXT: s_mov_b32 s4, s6
-; GFX11-NEXT: s_mov_b32 s5, s7
-; GFX11-NEXT: s_mov_b32 s6, s8
-; GFX11-NEXT: s_mov_b32 s7, s9
-; GFX11-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-NEXT: v_mov_b32_e32 v1, v6
-; GFX11-NEXT: v_mov_b32_e32 v2, v7
-; GFX11-NEXT: v_mov_b32_e32 v3, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe lwe
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v10
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX11-TRUE16-NEXT: image_load v[0:4], v[5:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe lwe
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe lwe
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v9
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v10
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX12-TRUE16-NEXT: image_load v[0:4], [v5, v6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: v_mov_b32_e32 v7, v5
-; GFX12-NEXT: v_mov_b32_e32 v8, v5
-; GFX12-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v11, v3, 16, v2
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-NEXT: v_mov_b32_e32 v1, v6
-; GFX12-NEXT: v_mov_b32_e32 v2, v7
-; GFX12-NEXT: v_mov_b32_e32 v3, v8
-; GFX12-NEXT: v_mov_b32_e32 v4, v9
-; GFX12-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX12-FAKE16-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 tfe
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
%v.vec = extractvalue { <4 x float>, i32 } %v, 0
%v.err = extractvalue { <4 x float>, i32 } %v, 1
@@ -314,3 +484,6 @@ declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i16(i32 immarg, i1
declare { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index e0b7c32d24993..2c0ee1740375a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -1,8 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw:
@@ -23,41 +25,89 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: load_3d_v4f32_xyzw:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10PLUS-NEXT: s_mov_b32 s0, s2
-; GFX10PLUS-NEXT: s_mov_b32 s1, s3
-; GFX10PLUS-NEXT: s_mov_b32 s2, s4
-; GFX10PLUS-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10PLUS-NEXT: s_mov_b32 s3, s5
-; GFX10PLUS-NEXT: s_mov_b32 s4, s6
-; GFX10PLUS-NEXT: s_mov_b32 s5, s7
-; GFX10PLUS-NEXT: s_mov_b32 s6, s8
-; GFX10PLUS-NEXT: s_mov_b32 s7, s9
-; GFX10PLUS-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: load_3d_v4f32_xyzw:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: image_load v[0:3], [v0, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_3d_v4f32_xyzw:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_3d_v4f32_xyzw:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: image_load v[0:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: image_load v[0:3], [v0, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: load_3d_v4f32_xyzw:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: image_load v[0:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
ret <4 x float> %v
}
@@ -122,57 +172,107 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX10-NEXT: global_store_dword v5, v4, s[10:11]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: load_3d_v4f32_xyzw_tfe:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: v_mov_b32_e32 v8, v5
-; GFX11-NEXT: v_mov_b32_e32 v9, v5
-; GFX11-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX11-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX11-NEXT: s_mov_b32 s2, s4
-; GFX11-NEXT: s_mov_b32 s3, s5
-; GFX11-NEXT: s_mov_b32 s4, s6
-; GFX11-NEXT: s_mov_b32 s5, s7
-; GFX11-NEXT: s_mov_b32 s6, s8
-; GFX11-NEXT: s_mov_b32 s7, s9
-; GFX11-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
-; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX11-TRUE16-NEXT: image_load v[0:4], v[5:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX12-TRUE16-NEXT: image_load v[0:4], [v5, v6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_3d_v4f32_xyzw_tfe:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v8, v5
-; GFX12-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
-; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT: v_mov_b32_e32 v4, v9
-; GFX12-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v9, v5
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v8, v5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v11, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX12-FAKE16-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue { <4 x float>, i32 } %v, 0
%v.err = extractvalue { <4 x float>, i32 } %v, 1
@@ -240,57 +340,107 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX10-NEXT: global_store_dword v5, v4, s[10:11]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: v_mov_b32_e32 v8, v5
-; GFX11-NEXT: v_mov_b32_e32 v9, v5
-; GFX11-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX11-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX11-NEXT: s_mov_b32 s2, s4
-; GFX11-NEXT: s_mov_b32 s3, s5
-; GFX11-NEXT: s_mov_b32 s4, s6
-; GFX11-NEXT: s_mov_b32 s5, s7
-; GFX11-NEXT: s_mov_b32 s6, s8
-; GFX11-NEXT: s_mov_b32 s7, s9
-; GFX11-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
-; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX11-NEXT: v_mov_b32_e32 v4, v9
-; GFX11-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe lwe
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX11-TRUE16-NEXT: image_load v[0:4], v[5:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe lwe
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, s0, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX11-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX11-FAKE16-NEXT: image_load v[0:4], v[10:11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe lwe
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s8
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s9
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v11
+; GFX12-TRUE16-NEXT: image_load v[0:4], [v5, v6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b32 v7, v4, s[10:11]
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_mov_b32 s0, s2
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v8, v5
-; GFX12-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX12-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX12-NEXT: s_mov_b32 s2, s4
-; GFX12-NEXT: s_mov_b32 s3, s5
-; GFX12-NEXT: s_mov_b32 s4, s6
-; GFX12-NEXT: s_mov_b32 s5, s7
-; GFX12-NEXT: s_mov_b32 s6, s8
-; GFX12-NEXT: s_mov_b32 s7, s9
-; GFX12-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
-; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT: v_mov_b32_e32 v4, v9
-; GFX12-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v5, v4, s[10:11]
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s1, s3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v9, v5
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v8, v5
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v10, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v11, s0, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, s6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s7
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s8
+; GFX12-FAKE16-NEXT: s_mov_b32 s7, s9
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v9
+; GFX12-FAKE16-NEXT: image_load v[0:4], [v10, v11], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D a16 tfe
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b32 v5, v4, s[10:11]
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
%v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 3, i32 0)
%v.vec = extractvalue { <4 x float>, i32 } %v, 0
%v.err = extractvalue { <4 x float>, i32 } %v, 1
@@ -302,3 +452,6 @@ declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 immarg, i16, i16, i
declare { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 400ca7ca664e4..b0742121ff5ec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -1,7 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_d_1d:
@@ -35,21 +37,37 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_d_2d:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_d_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_d_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_d_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_d_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_d_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f16.f32(i32 15, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <4 x float> %v
@@ -66,21 +84,37 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_d_3d:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_d_3d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_d_3d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_d_3d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_d_3d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_d_3d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f16.f32(i32 15, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <4 x float> %v
@@ -118,21 +152,37 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_c_d_2d:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX11-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_c_d_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_c_d_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_c_d_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_c_d_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_c_d_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f16.f32(i32 15, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <4 x float> %v
@@ -170,21 +220,37 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_d_cl_2d:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_d_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_d_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_d_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_d_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_d_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f16.f32(i32 15, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <4 x float> %v
@@ -224,21 +290,37 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_c_d_cl_2d:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_c_d_cl_2d:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_c_d_cl_2d:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_c_d_cl_2d:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_c_d_cl_2d:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_c_d_cl_2d:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f16.f32(i32 15, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <4 x float> %v
@@ -257,21 +339,39 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_c_d_o_2darray_V1:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_c_d_o_2darray_V1:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_c_d_o_2darray_V1:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_c_d_o_2darray_V1:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f16.f32.f32(i32 4, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret float %v
@@ -290,21 +390,39 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: sample_c_d_o_2darray_V2:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
+; GFX11-TRUE16: ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: sample_c_d_o_2darray_V2:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: sample_c_d_o_2darray_V2:
+; GFX11-FAKE16: ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX11-FAKE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
+; GFX12-TRUE16: ; %bb.0: ; %main_body
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: sample_c_d_o_2darray_V2:
+; GFX12-FAKE16: ; %bb.0: ; %main_body
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-FAKE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
main_body:
%v = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f32(i32 6, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret <2 x float> %v
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
index 302cf1d691436..92461b4d508d4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -4,8 +4,10 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=+real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11,GFX11-TRUE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=+real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12,GFX12-TRUE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
@@ -111,27 +113,27 @@ define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i8_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i8), addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i8) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_i8_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i8), addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i8) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i8_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -242,50 +244,101 @@ define amdgpu_ps void @raw_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
- ; GFX11-LABEL: name: raw_buffer_load_i16_tfe
- ; GFX11: bb.1 (%ir-block.0):
- ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
- ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
- ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
- ; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX11-NEXT: S_ENDPGM 0
+ ; GFX11-TRUE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX11-TRUE16: bb.1 (%ir-block.0):
+ ; GFX11-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-TRUE16-NEXT: {{ $}}
+ ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-TRUE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX11-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX11-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX11-TRUE16-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX11-TRUE16-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX11-TRUE16-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[COPY10]]
+ ; GFX11-TRUE16-NEXT: GLOBAL_STORE_SHORT_t16 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX11-FAKE16: bb.1 (%ir-block.0):
+ ; GFX11-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-FAKE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX11-FAKE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX11-FAKE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-TRUE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX12-TRUE16: bb.1 (%ir-block.0):
+ ; GFX12-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-TRUE16-NEXT: {{ $}}
+ ; GFX12-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-TRUE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX12-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-TRUE16-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-TRUE16-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-TRUE16-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[COPY10]]
+ ; GFX12-TRUE16-NEXT: GLOBAL_STORE_SHORT_t16 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i16_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-FAKE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX12-FAKE16: bb.1 (%ir-block.0):
+ ; GFX12-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-FAKE16-NEXT: {{ $}}
+ ; GFX12-FAKE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-FAKE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-FAKE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-FAKE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX12-FAKE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-FAKE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-FAKE16-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i16_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -392,55 +445,115 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
- ; GFX11-LABEL: name: raw_buffer_load_f16_tfe
- ; GFX11: bb.1 (%ir-block.0):
- ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX11-NEXT: S_ENDPGM 0
+ ; GFX11-TRUE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX11-TRUE16: bb.1 (%ir-block.0):
+ ; GFX11-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-TRUE16-NEXT: {{ $}}
+ ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
+ ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; GFX11-TRUE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
+ ; GFX11-TRUE16-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
+ ; GFX11-TRUE16-NEXT: [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11-TRUE16-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX11-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[COPY8]](i32), [[COPY9]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-TRUE16-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
+ ; GFX11-TRUE16-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<2 x i32>) = G_BUILD_VECTOR [[AMDGPU_READANYLANE]](i32), [[AMDGPU_READANYLANE1]](i32)
+ ; GFX11-TRUE16-NEXT: [[UV2:%[0-9]+]]:sgpr(i32), [[UV3:%[0-9]+]]:sgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX11-TRUE16-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[UV2]](i32)
+ ; GFX11-TRUE16-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-TRUE16-NEXT: G_STORE [[BITCAST]](f16), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: G_STORE [[UV3]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX11-FAKE16: bb.1 (%ir-block.0):
+ ; GFX11-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11-FAKE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11-FAKE16-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX11-FAKE16-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX11-FAKE16-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11-FAKE16-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX11-FAKE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-FAKE16-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX11-FAKE16-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
+ ; GFX11-FAKE16-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-FAKE16-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
+ ; GFX11-FAKE16-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-TRUE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX12-TRUE16: bb.1 (%ir-block.0):
+ ; GFX12-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-TRUE16-NEXT: {{ $}}
+ ; GFX12-TRUE16-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
+ ; GFX12-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; GFX12-TRUE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
+ ; GFX12-TRUE16-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
+ ; GFX12-TRUE16-NEXT: [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX12-TRUE16-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX12-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[COPY8]](i32), [[COPY9]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX12-TRUE16-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
+ ; GFX12-TRUE16-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<2 x i32>) = G_BUILD_VECTOR [[AMDGPU_READANYLANE]](i32), [[AMDGPU_READANYLANE1]](i32)
+ ; GFX12-TRUE16-NEXT: [[UV2:%[0-9]+]]:sgpr(i32), [[UV3:%[0-9]+]]:sgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX12-TRUE16-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[UV2]](i32)
+ ; GFX12-TRUE16-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-TRUE16-NEXT: G_STORE [[BITCAST]](f16), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: G_STORE [[UV3]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-FAKE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX12-FAKE16: bb.1 (%ir-block.0):
+ ; GFX12-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-FAKE16-NEXT: {{ $}}
+ ; GFX12-FAKE16-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
+ ; GFX12-FAKE16-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX12-FAKE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX12-FAKE16-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX12-FAKE16-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX12-FAKE16-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX12-FAKE16-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX12-FAKE16-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX12-FAKE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX12-FAKE16-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX12-FAKE16-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
+ ; GFX12-FAKE16-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-FAKE16-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
+ ; GFX12-FAKE16-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -577,27 +690,27 @@ define amdgpu_ps void @raw_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i32), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i32) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i32), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (i32) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -751,32 +864,32 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v2i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v2i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v2i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -935,32 +1048,32 @@ define amdgpu_ps void @raw_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v2f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x f32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v2f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x f32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v2f32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1176,34 +1289,34 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v3i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v3i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v3i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1419,34 +1532,34 @@ define amdgpu_ps void @raw_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v3f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x f32>) into %ir.data_addr, align 16, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v3f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x f32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v3f32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1623,36 +1736,36 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v4i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v4i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v4i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1831,36 +1944,36 @@ define amdgpu_ps void @raw_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v4f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x f32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v4f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x f32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v4f32_tfe
; GFX1250: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
index 0d5b69d01a3c5..58396f9c79990 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -4,8 +4,10 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=+real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11,GFX11-TRUE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=+real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12,GFX12-TRUE16
+; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
@@ -115,29 +117,29 @@ define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i8_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i8), addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i8) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_i8_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i8), addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i8) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i8_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -253,53 +255,107 @@ define amdgpu_ps void @raw_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
- ; GFX11-LABEL: name: raw_buffer_load_i16_tfe
- ; GFX11: bb.1 (%ir-block.0):
- ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
- ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
- ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
- ; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX11-NEXT: S_ENDPGM 0
+ ; GFX11-TRUE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX11-TRUE16: bb.1 (%ir-block.0):
+ ; GFX11-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-TRUE16-NEXT: {{ $}}
+ ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-TRUE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX11-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX11-TRUE16-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX11-TRUE16-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX11-TRUE16-NEXT: [[COPY11:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX11-TRUE16-NEXT: [[COPY12:%[0-9]+]]:vgpr_16 = COPY [[COPY11]]
+ ; GFX11-TRUE16-NEXT: GLOBAL_STORE_SHORT_t16 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX11-FAKE16: bb.1 (%ir-block.0):
+ ; GFX11-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-FAKE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-FAKE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX11-FAKE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX11-FAKE16-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-TRUE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX12-TRUE16: bb.1 (%ir-block.0):
+ ; GFX12-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-TRUE16-NEXT: {{ $}}
+ ; GFX12-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-TRUE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-TRUE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX12-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-TRUE16-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-TRUE16-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-TRUE16-NEXT: [[COPY11:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-TRUE16-NEXT: [[COPY12:%[0-9]+]]:vgpr_16 = COPY [[COPY11]]
+ ; GFX12-TRUE16-NEXT: GLOBAL_STORE_SHORT_t16 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i16_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-FAKE16-LABEL: name: raw_buffer_load_i16_tfe
+ ; GFX12-FAKE16: bb.1 (%ir-block.0):
+ ; GFX12-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-FAKE16-NEXT: {{ $}}
+ ; GFX12-FAKE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-FAKE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-FAKE16-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-FAKE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-FAKE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-FAKE16-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; GFX12-FAKE16-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-FAKE16-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-FAKE16-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i16) into %ir.data_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i16_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -408,55 +464,115 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
- ; GFX11-LABEL: name: raw_buffer_load_f16_tfe
- ; GFX11: bb.1 (%ir-block.0):
- ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX11-NEXT: S_ENDPGM 0
+ ; GFX11-TRUE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX11-TRUE16: bb.1 (%ir-block.0):
+ ; GFX11-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-TRUE16-NEXT: {{ $}}
+ ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
+ ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; GFX11-TRUE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
+ ; GFX11-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
+ ; GFX11-TRUE16-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
+ ; GFX11-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
+ ; GFX11-TRUE16-NEXT: [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11-TRUE16-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX11-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX11-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[COPY8]](i32), [[COPY9]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-TRUE16-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
+ ; GFX11-TRUE16-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
+ ; GFX11-TRUE16-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<2 x i32>) = G_BUILD_VECTOR [[AMDGPU_READANYLANE]](i32), [[AMDGPU_READANYLANE1]](i32)
+ ; GFX11-TRUE16-NEXT: [[UV2:%[0-9]+]]:sgpr(i32), [[UV3:%[0-9]+]]:sgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX11-TRUE16-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[UV2]](i32)
+ ; GFX11-TRUE16-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-TRUE16-NEXT: G_STORE [[BITCAST]](f16), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: G_STORE [[UV3]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX11-FAKE16: bb.1 (%ir-block.0):
+ ; GFX11-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11-FAKE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11-FAKE16-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX11-FAKE16-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX11-FAKE16-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX11-FAKE16-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11-FAKE16-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX11-FAKE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-FAKE16-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX11-FAKE16-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
+ ; GFX11-FAKE16-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-FAKE16-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
+ ; GFX11-FAKE16-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-TRUE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX12-TRUE16: bb.1 (%ir-block.0):
+ ; GFX12-TRUE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-TRUE16-NEXT: {{ $}}
+ ; GFX12-TRUE16-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
+ ; GFX12-TRUE16-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; GFX12-TRUE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
+ ; GFX12-TRUE16-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
+ ; GFX12-TRUE16-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
+ ; GFX12-TRUE16-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
+ ; GFX12-TRUE16-NEXT: [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX12-TRUE16-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX12-TRUE16-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX12-TRUE16-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[COPY8]](i32), [[COPY9]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX12-TRUE16-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
+ ; GFX12-TRUE16-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
+ ; GFX12-TRUE16-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<2 x i32>) = G_BUILD_VECTOR [[AMDGPU_READANYLANE]](i32), [[AMDGPU_READANYLANE1]](i32)
+ ; GFX12-TRUE16-NEXT: [[UV2:%[0-9]+]]:sgpr(i32), [[UV3:%[0-9]+]]:sgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX12-TRUE16-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[UV2]](i32)
+ ; GFX12-TRUE16-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-TRUE16-NEXT: G_STORE [[BITCAST]](f16), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: G_STORE [[UV3]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-TRUE16-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-FAKE16-LABEL: name: raw_buffer_load_f16_tfe
+ ; GFX12-FAKE16: bb.1 (%ir-block.0):
+ ; GFX12-FAKE16-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-FAKE16-NEXT: {{ $}}
+ ; GFX12-FAKE16-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
+ ; GFX12-FAKE16-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX12-FAKE16-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX12-FAKE16-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX12-FAKE16-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX12-FAKE16-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX12-FAKE16-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX12-FAKE16-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX12-FAKE16-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX12-FAKE16-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX12-FAKE16-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX12-FAKE16-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
+ ; GFX12-FAKE16-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
+ ; GFX12-FAKE16-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-FAKE16-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
+ ; GFX12-FAKE16-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-FAKE16-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -597,29 +713,29 @@ define amdgpu_ps void @raw_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i32), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (i32), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -779,34 +895,34 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v2i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v2i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v2i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -971,34 +1087,34 @@ define amdgpu_ps void @raw_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v2f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x f32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v2f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x f32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v2f32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1221,36 +1337,36 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v3i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v3i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v3i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1473,36 +1589,36 @@ define amdgpu_ps void @raw_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v3f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x f32>) into %ir.data_addr, align 16, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v3f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x f32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v3f32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1685,38 +1801,38 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v4i32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x i32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v4i32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x i32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v4i32_tfe
; GFX1250: bb.1 (%ir-block.0):
@@ -1901,38 +2017,38 @@ define amdgpu_ps void @raw_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
- ; GFX1200-LABEL: name: raw_buffer_load_v4f32_tfe
- ; GFX1200: bb.1 (%ir-block.0):
- ; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
- ; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1200-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f32>), align 1, addrspace 8)
- ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
- ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
- ; GFX1200-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1200-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
- ; GFX1200-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1200-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX1200-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1200-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX1200-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x f32>) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
- ; GFX1200-NEXT: S_ENDPGM 0
+ ; GFX12-LABEL: name: raw_buffer_load_v4f32_tfe
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f32>), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x f32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_v4f32_tfe
; GFX1250: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index c79009e14e669..1d42bc3cf14f6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -3,7 +3,8 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i8 @v_lshr_i8(i8 %value, i8 %amount) {
; GFX6-LABEL: v_lshr_i8:
@@ -26,13 +27,29 @@ define i8 @v_lshr_i8(i8 %value, i8 %amount) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_lshr_i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10PLUS-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, v1, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_lshr_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_lshr_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_lshr_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = lshr i8 %value, %amount
ret i8 %result
}
@@ -58,12 +75,26 @@ define i8 @v_lshr_i8_7(i8 %value) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_lshr_i8_7:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, 7, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_lshr_i8_7:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: v_lshrrev_b16 v0, 7, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_lshr_i8_7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 7, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_lshr_i8_7:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 7, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = lshr i8 %value, 7
ret i8 %result
}
@@ -615,11 +646,23 @@ define i16 @v_lshr_i16(i16 %value, i16 %amount) {
; GFX9-NEXT: v_lshrrev_b16_e32 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_lshr_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, v1, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_lshr_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_lshr_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_lshr_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = lshr i16 %value, %amount
ret i16 %result
}
@@ -643,11 +686,23 @@ define i16 @v_lshr_i16_15(i16 %value) {
; GFX9-NEXT: v_lshrrev_b16_e32 v0, 15, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_lshr_i16_15:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, 15, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_lshr_i16_15:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshrrev_b16 v0, 15, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_lshr_i16_15:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 15, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_lshr_i16_15:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 15, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = lshr i16 %value, 15
ret i16 %result
}
@@ -724,10 +779,20 @@ define amdgpu_ps half @lshr_i16_sv(i16 inreg %value, i16 %amount) {
; GFX9-NEXT: v_lshrrev_b16_e64 v0, v0, s0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: lshr_i16_sv:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, v0, s0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: lshr_i16_sv:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshrrev_b16 v0, v0, s0
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: lshr_i16_sv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: lshr_i16_sv:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = lshr i16 %value, %amount
%cast = bitcast i16 %result to half
ret half %cast
@@ -750,10 +815,20 @@ define amdgpu_ps half @lshr_i16_vs(i16 %value, i16 inreg %amount) {
; GFX9-NEXT: v_lshrrev_b16_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: lshr_i16_vs:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_lshrrev_b16 v0, s0, v0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: lshr_i16_vs:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshrrev_b16 v0, s0, v0
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: lshr_i16_vs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, s0, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: lshr_i16_vs:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, s0, v0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = lshr i16 %value, %amount
%cast = bitcast i16 %result to half
ret half %cast
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 3e70134a4a303..4594f97385e47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -3,9 +3,10 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
; RUN: llc -global-isel -mtriple=amdgpu13.10 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX13 %s
@@ -80,15 +81,25 @@ define i16 @v_mul_i16(i16 %num, i16 %den) {
; GFX11-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: v_mul_i16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: v_mul_i16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: v_mul_i16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_mul_i16:
; GFX1250: ; %bb.0:
@@ -203,17 +214,29 @@ define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: v_mul_i16_zeroext:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: v_mul_i16_zeroext:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: v_mul_i16_zeroext:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_mul_i16_zeroext:
; GFX1250: ; %bb.0:
@@ -324,17 +347,29 @@ define signext i16 @v_mul_i16_signext(i16 signext %num, i16 signext %den) {
; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: v_mul_i16_signext:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: v_mul_i16_signext:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: v_mul_i16_signext:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_mul_i16_signext:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index 50d94aa295aa6..f55aa1a950390 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -3,7 +3,9 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+
define i8 @v_shl_i8(i8 %value, i8 %amount) {
; GFX6-LABEL: v_shl_i8:
@@ -25,12 +27,26 @@ define i8 @v_shl_i8(i8 %value, i8 %amount) {
; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_shl_i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_shl_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_shl_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_shl_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = shl i8 %value, %amount
ret i8 %result
}
@@ -54,11 +70,23 @@ define i8 @v_shl_i8_7(i8 %value) {
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 7, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_shl_i8_7:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, 7, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_shl_i8_7:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b16 v0, 7, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_shl_i8_7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 7, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_shl_i8_7:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 7, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = shl i8 %value, 7
ret i8 %result
}
@@ -582,11 +610,23 @@ define i16 @v_shl_i16(i16 %value, i16 %amount) {
; GFX9-NEXT: v_lshlrev_b16_e32 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_shl_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_shl_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_shl_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_shl_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = shl i16 %value, %amount
ret i16 %result
}
@@ -610,11 +650,23 @@ define i16 @v_shl_i16_15(i16 %value) {
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 15, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_shl_i16_15:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, 15, v0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_shl_i16_15:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b16 v0, 15, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_shl_i16_15:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 15, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_shl_i16_15:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 15, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = shl i16 %value, 15
ret i16 %result
}
@@ -663,10 +715,20 @@ define amdgpu_ps half @shl_i16_sv(i16 inreg %value, i16 %amount) {
; GFX9-NEXT: v_lshlrev_b16_e64 v0, v0, s0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: shl_i16_sv:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, v0, s0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: shl_i16_sv:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshlrev_b16 v0, v0, s0
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: shl_i16_sv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: shl_i16_sv:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = shl i16 %value, %amount
%cast = bitcast i16 %result to half
ret half %cast
@@ -688,10 +750,20 @@ define amdgpu_ps half @shl_i16_vs(i16 %value, i16 inreg %amount) {
; GFX9-NEXT: v_lshlrev_b16_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: shl_i16_vs:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_lshlrev_b16 v0, s0, v0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: shl_i16_vs:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshlrev_b16 v0, s0, v0
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: shl_i16_vs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s0, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: shl_i16_vs:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, s0, v0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = shl i16 %value, %amount
%cast = bitcast i16 %result to half
ret half %cast
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index 33cdab2bdc632..8241d2b7109dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170
-; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12
+; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170,GFX1170-TRUE16
+; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170,GFX1170-FAKE16
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-TRUE16
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-FAKE16
define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GCN-LABEL: test_wmma_f32_16x16x16_f16_negA:
@@ -405,47 +407,85 @@ bb:
; pack f16 elements with v_perm_b32 since they don't come from same b32
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x half> %B, ptr %Caddr, ptr addrspace(1) %out) {
-; GFX1170-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
-; GFX1170: ; %bb.0: ; %bb
-; GFX1170-NEXT: s_clause 0x1
-; GFX1170-NEXT: flat_load_b128 v[12:15], v[8:9]
-; GFX1170-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
-; GFX1170-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
-; GFX1170-NEXT: v_and_b32_e32 v8, 0xffff, v12
-; GFX1170-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX1170-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v14, 0xffff, v16
-; GFX1170-NEXT: v_and_b32_e32 v16, 0xffff, v18
-; GFX1170-NEXT: v_lshl_or_b32 v12, v13, 16, v8
-; GFX1170-NEXT: v_lshl_or_b32 v13, v15, 16, v9
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1170-NEXT: v_lshl_or_b32 v14, v17, 16, v14
-; GFX1170-NEXT: v_lshl_or_b32 v15, v19, 16, v16
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
-; GFX1170-NEXT: global_store_b128 v[10:11], v[12:15], off
-; GFX1170-NEXT: s_endpgm
+; GFX1170-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX1170-TRUE16: ; %bb.0: ; %bb
+; GFX1170-TRUE16-NEXT: s_clause 0x1
+; GFX1170-TRUE16-NEXT: flat_load_b128 v[12:15], v[8:9]
+; GFX1170-TRUE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
+; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-TRUE16-NEXT: v_dual_mov_b32 v13, v14 :: v_dual_mov_b32 v14, v16
+; GFX1170-TRUE16-NEXT: v_mov_b32_e32 v15, v18
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX1170-TRUE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX1170-TRUE16-NEXT: s_endpgm
;
-; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: flat_load_b128 v[12:15], v[8:9]
-; GFX12-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x101
-; GFX12-NEXT: v_and_b32_e32 v8, 0xffff, v12
-; GFX12-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v14, 0xffff, v16
-; GFX12-NEXT: v_and_b32_e32 v16, 0xffff, v18
-; GFX12-NEXT: v_lshl_or_b32 v12, v13, 16, v8
-; GFX12-NEXT: v_lshl_or_b32 v13, v15, 16, v9
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_lshl_or_b32 v14, v17, 16, v14
-; GFX12-NEXT: v_lshl_or_b32 v15, v19, 16, v16
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
-; GFX12-NEXT: global_store_b128 v[10:11], v[12:15], off
-; GFX12-NEXT: s_endpgm
+; GFX1170-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX1170-FAKE16: ; %bb.0: ; %bb
+; GFX1170-FAKE16-NEXT: s_clause 0x1
+; GFX1170-FAKE16-NEXT: flat_load_b128 v[12:15], v[8:9]
+; GFX1170-FAKE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX1170-FAKE16-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v14
+; GFX1170-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v16
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v18
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v12, v13, 16, v8
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v13, v15, 16, v9
+; GFX1170-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v14, v17, 16, v14
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v15, v19, 16, v16
+; GFX1170-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX1170-FAKE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX1170-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: flat_load_b128 v[12:15], v[8:9]
+; GFX12-TRUE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, v14 :: v_dual_mov_b32 v14, v16
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v15, v18
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX12-TRUE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: flat_load_b128 v[12:15], v[8:9]
+; GFX12-FAKE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v14
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v18
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v12, v13, 16, v8
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v13, v15, 16, v9
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v14, v17, 16, v14
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v15, v19, 16, v16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX12-FAKE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX12-FAKE16-NEXT: s_endpgm
bb:
%C = load <16 x half>, ptr %Caddr
%C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -468,3 +508,6 @@ declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v2i32.v8f32(<2 x i32>
declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v2i32.v8f32(<2 x i32>, <2 x i32>, <8 x float>)
declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v8f16.v16f16.v8f32.i16(<8 x half>, <16 x half>, <8 x float>, i16)
declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v8f16.v16f16.v8f16.i16(<8 x half>, <16 x half>, <8 x half>, i16)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1170: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
index 5839534a5dc78..2b84ad83aab50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=+wavefrontsize64,-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170
-; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+wavefrontsize64,-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12
+; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=+wavefrontsize64,+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170,GFX1170-TRUE16
+; RUN: llc -global-isel -mtriple=amdgpu11.70 -mattr=+wavefrontsize64,-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX1170,GFX1170-FAKE16
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+wavefrontsize64,+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-TRUE16
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+wavefrontsize64,-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-FAKE16
define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {
; GCN-LABEL: test_wmma_f32_16x16x16_f16_negA:
@@ -367,33 +369,57 @@ bb:
; pack f16 elements with v_perm_b32 since they don't come from same b32
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<4 x half> %A, <4 x half> %B, ptr %Caddr, ptr addrspace(1) %out) {
-; GFX1170-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
-; GFX1170: ; %bb.0: ; %bb
-; GFX1170-NEXT: flat_load_b128 v[8:11], v[4:5]
-; GFX1170-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; GFX1170-NEXT: v_and_b32_e32 v5, 0xffff, v10
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1170-NEXT: v_lshl_or_b32 v4, v9, 16, v4
-; GFX1170-NEXT: v_lshl_or_b32 v5, v11, 16, v5
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]
-; GFX1170-NEXT: global_store_b64 v[6:7], v[4:5], off
-; GFX1170-NEXT: s_endpgm
+; GFX1170-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX1170-TRUE16: ; %bb.0: ; %bb
+; GFX1170-TRUE16-NEXT: flat_load_b128 v[8:11], v[4:5]
+; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v10.h, v11.l
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_mov_b32_e32 v9, v10
+; GFX1170-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[8:9], v[0:1], v[2:3], v[8:9] neg_lo:[0,0,1]
+; GFX1170-TRUE16-NEXT: global_store_b64 v[6:7], v[8:9], off
+; GFX1170-TRUE16-NEXT: s_endpgm
;
-; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: flat_load_b128 v[8:11], v[4:5]
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v10
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_lshl_or_b32 v4, v9, 16, v4
-; GFX12-NEXT: v_lshl_or_b32 v5, v11, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]
-; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off
-; GFX12-NEXT: s_endpgm
+; GFX1170-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX1170-FAKE16: ; %bb.0: ; %bb
+; GFX1170-FAKE16-NEXT: flat_load_b128 v[8:11], v[4:5]
+; GFX1170-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v8
+; GFX1170-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; GFX1170-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v4, v9, 16, v4
+; GFX1170-FAKE16-NEXT: v_lshl_or_b32 v5, v11, 16, v5
+; GFX1170-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]
+; GFX1170-FAKE16-NEXT: global_store_b64 v[6:7], v[4:5], off
+; GFX1170-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: flat_load_b128 v[8:11], v[4:5]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.h, v11.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v10
+; GFX12-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[8:9], v[0:1], v[2:3], v[8:9] neg_lo:[0,0,1]
+; GFX12-TRUE16-NEXT: global_store_b64 v[6:7], v[8:9], off
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: flat_load_b128 v[8:11], v[4:5]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v8
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v4, v9, 16, v4
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v5, v11, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]
+; GFX12-FAKE16-NEXT: global_store_b64 v[6:7], v[4:5], off
+; GFX12-FAKE16-NEXT: s_endpgm
bb:
%C = load <8 x half>, ptr %Caddr
%C_shuffle = shufflevector <8 x half> %C, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -416,3 +442,6 @@ declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.i32.v4f32(i32, i32, <
declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.i32.v4f32(i32, i32, <4 x float>)
declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f16.v8f16.v4f32.i16(<4 x half>, <8 x half>, <4 x float>, i16)
declare <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v8f16.v4f16.i16(<4 x half>, <8 x half>, <4 x half>, i16)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1170: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
index 6f921981ac212..ae79ef5dcad58 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
@@ -146,33 +146,6 @@ define amdgpu_kernel void @fadd_f16(
; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[10:11]
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_f16:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
-; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
-; GFX11-NEXT: s_mov_b32 s11, 0x31016000
-; GFX11-NEXT: s_mov_b32 s10, -1
-; GFX11-NEXT: s_mov_b32 s3, s11
-; GFX11-NEXT: s_mov_b32 s2, s10
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s8, s4
-; GFX11-NEXT: s_mov_b32 s9, s5
-; GFX11-NEXT: s_mov_b32 s4, s6
-; GFX11-NEXT: s_mov_b32 s5, s7
-; GFX11-NEXT: s_mov_b32 s6, s10
-; GFX11-NEXT: s_mov_b32 s7, s11
-; GFX11-NEXT: buffer_load_u16 v0, off, s[4:7], 0 glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_load_u16 v1, off, s[0:3], 0 glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: buffer_store_b16 v0, off, s[8:11], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b) {
@@ -288,27 +261,6 @@ define amdgpu_kernel void @fadd_f16_imm_a(
; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_f16_imm_a:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: s_mov_b32 s2, s6
-; GFX11-NEXT: s_mov_b32 s3, s7
-; GFX11-NEXT: buffer_load_u16 v0, off, s[0:3], 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v0.h, 0x3c00
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %b) {
entry:
@@ -422,27 +374,6 @@ define amdgpu_kernel void @fadd_f16_imm_b(
; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_f16_imm_b:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_mov_b32 s0, s2
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: s_mov_b32 s2, s6
-; GFX11-NEXT: s_mov_b32 s3, s7
-; GFX11-NEXT: buffer_load_u16 v0, off, s[0:3], 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v0.h, 0x4000
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
entry:
@@ -591,26 +522,6 @@ define amdgpu_kernel void @fadd_v2f16(
; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, v1, v0
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_v2f16:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
-; GFX11-NEXT: s_load_b64 s[8:9], s[0:1], 0x34
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b32 v1, v0, s[6:7]
-; GFX11-NEXT: global_load_b32 v0, v0, s[8:9]
-; GFX11-NEXT: s_mov_b32 s0, s4
-; GFX11-NEXT: s_mov_b32 s1, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v0, v1, v0
-; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b) {
@@ -737,22 +648,6 @@ define amdgpu_kernel void @fadd_v2f16_imm_a(
; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, 0x40003c00, v0
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_v2f16_imm_a:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v0, 0x40003c00, v0
-; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %b) {
entry:
@@ -876,22 +771,6 @@ define amdgpu_kernel void @fadd_v2f16_imm_b(
; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, 0x3c004000, v0
; GFX11-FAKE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
-; GFX11-LABEL: fadd_v2f16_imm_b:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v0, 0x3c004000, v0
-; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0
-; GFX11-NEXT: s_nop 0
-; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma-fake16.mir b/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma-fake16.mir
new file mode 100644
index 0000000000000..ddf3aa2e17ca4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma-fake16.mir
@@ -0,0 +1,102 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 %s -run-pass twoaddressinstruction -verify-machineinstrs -o - | FileCheck --check-prefixes=GFX11 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 %s --passes=two-address-instruction -verify-each -o - | FileCheck --check-prefixes=GFX11 %s
+
+---
+name: test_fmamk_reg_imm_f16
+registers:
+ - { id: 0, class: vreg_64 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: vgpr_32 }
+ - { id: 4, class: vgpr_32 }
+body: |
+ bb.0:
+
+ ; GFX11-LABEL: name: test_fmamk_reg_imm_f16
+ ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
+ ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, killed [[COPY1]], 0, [[V_MOV_B32_e32_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ %0 = IMPLICIT_DEF
+ %1 = COPY %0.sub1
+ %2 = COPY %0.sub0
+ %3 = V_MOV_B32_e32 1078523331, implicit $exec
+ %4 = V_FMAC_F16_fake16_e64 0, killed %2, 0, %3, 0, killed %1, 0, 0, implicit $mode, implicit $exec
+
+...
+
+---
+name: test_fmamk_imm_reg_f16
+registers:
+ - { id: 0, class: vreg_64 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: vgpr_32 }
+ - { id: 4, class: vgpr_32 }
+body: |
+ bb.0:
+
+ ; GFX11-LABEL: name: test_fmamk_imm_reg_f16
+ ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
+ ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, [[COPY1]], 0, killed [[V_MOV_B32_e32_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ %0 = IMPLICIT_DEF
+ %1 = COPY %0.sub1
+ %2 = COPY %0.sub0
+ %3 = V_MOV_B32_e32 1078523331, implicit $exec
+ %4 = V_FMAC_F16_fake16_e64 0, %2, 0, killed %3, 0, killed %1, 0, 0, implicit $mode, implicit $exec
+
+...
+
+---
+name: test_fmaak_f16
+registers:
+ - { id: 0, class: vreg_64 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: vgpr_32 }
+ - { id: 4, class: vgpr_32 }
+body: |
+ bb.0:
+
+ ; GFX11-LABEL: name: test_fmaak_f16
+ ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
+ ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, killed [[COPY]], 0, [[COPY1]], 0, [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
+ %0 = IMPLICIT_DEF
+ %1 = COPY %0.sub0
+ %2 = COPY %0.sub1
+ %3 = V_MOV_B32_e32 1078523331, implicit $exec
+ %4 = V_FMAC_F16_fake16_e64 0, killed %1, 0, %2, 0, %3, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: test_fmaak_inline_literal_f16
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0', virtual-reg: '%0' }
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; GFX11-LABEL: name: test_fmaak_inline_literal_f16
+ ; GFX11: liveins: $vgpr0
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+ ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, 16384, 0, killed [[COPY]], 0, [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY killed $vgpr0
+
+ %1:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
+ %2:vgpr_32 = V_FMAC_F16_fake16_e64 0, 16384, 0, killed %0, 0, %1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma.mir b/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma.mir
index acea2e501283b..b4c9be4a2f928 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-twoaddr-fma.mir
@@ -1,86 +1,86 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 %s -run-pass twoaddressinstruction -verify-machineinstrs -o - | FileCheck --check-prefixes=GFX11 %s
-# RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 %s --passes=two-address-instruction -verify-each -o - | FileCheck --check-prefixes=GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 %s -run-pass twoaddressinstruction -verify-machineinstrs -o - | FileCheck --check-prefixes=GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 %s --passes=two-address-instruction -verify-each -o - | FileCheck --check-prefixes=GFX11 %s
---
name: test_fmamk_reg_imm_f16
registers:
- - { id: 0, class: vreg_64 }
- - { id: 1, class: vgpr_32 }
- - { id: 2, class: vgpr_32 }
- - { id: 3, class: vgpr_32 }
- - { id: 4, class: vgpr_32 }
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vgpr_16 }
+ - { id: 2, class: vgpr_16 }
+ - { id: 3, class: vgpr_16 }
+ - { id: 4, class: vgpr_16 }
body: |
bb.0:
; GFX11-LABEL: name: test_fmamk_reg_imm_f16
- ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
- ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
- ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, killed [[COPY1]], 0, [[V_MOV_B32_e32_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY [[DEF]].hi16
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[DEF]].lo16
+ ; GFX11-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_t16_e64_:%[0-9]+]]:vgpr_16 = V_FMA_F16_gfx9_t16_e64 0, killed [[COPY1]], 0, [[V_MOV_B16_t16_e64_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
%0 = IMPLICIT_DEF
- %1 = COPY %0.sub1
- %2 = COPY %0.sub0
- %3 = V_MOV_B32_e32 1078523331, implicit $exec
- %4 = V_FMAC_F16_fake16_e64 0, killed %2, 0, %3, 0, killed %1, 0, 0, implicit $mode, implicit $exec
+ %1 = COPY %0.hi16
+ %2 = COPY %0.lo16
+ %3 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ %4 = V_FMAC_F16_t16_e64 0, killed %2, 0, %3, 0, killed %1, 0, 0, 0, implicit $mode, implicit $exec
...
---
name: test_fmamk_imm_reg_f16
registers:
- - { id: 0, class: vreg_64 }
- - { id: 1, class: vgpr_32 }
- - { id: 2, class: vgpr_32 }
- - { id: 3, class: vgpr_32 }
- - { id: 4, class: vgpr_32 }
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vgpr_16 }
+ - { id: 2, class: vgpr_16 }
+ - { id: 3, class: vgpr_16 }
+ - { id: 4, class: vgpr_16 }
body: |
bb.0:
; GFX11-LABEL: name: test_fmamk_imm_reg_f16
- ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
- ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
- ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, [[COPY1]], 0, killed [[V_MOV_B32_e32_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY [[DEF]].hi16
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[DEF]].lo16
+ ; GFX11-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_t16_e64_:%[0-9]+]]:vgpr_16 = V_FMA_F16_gfx9_t16_e64 0, [[COPY1]], 0, killed [[V_MOV_B16_t16_e64_]], 0, killed [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
%0 = IMPLICIT_DEF
- %1 = COPY %0.sub1
- %2 = COPY %0.sub0
- %3 = V_MOV_B32_e32 1078523331, implicit $exec
- %4 = V_FMAC_F16_fake16_e64 0, %2, 0, killed %3, 0, killed %1, 0, 0, implicit $mode, implicit $exec
+ %1 = COPY %0.hi16
+ %2 = COPY %0.lo16
+ %3 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ %4 = V_FMAC_F16_t16_e64 0, %2, 0, killed %3, 0, killed %1, 0, 0, 0, implicit $mode, implicit $exec
...
---
name: test_fmaak_f16
registers:
- - { id: 0, class: vreg_64 }
- - { id: 1, class: vgpr_32 }
- - { id: 2, class: vgpr_32 }
- - { id: 3, class: vgpr_32 }
- - { id: 4, class: vgpr_32 }
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vgpr_16 }
+ - { id: 2, class: vgpr_16 }
+ - { id: 3, class: vgpr_16 }
+ - { id: 4, class: vgpr_16 }
body: |
bb.0:
; GFX11-LABEL: name: test_fmaak_f16
- ; GFX11: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub1
- ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1078523331, implicit $exec
- ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, killed [[COPY]], 0, [[COPY1]], 0, [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY [[DEF]].hi16
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[DEF]].lo16
+ ; GFX11-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_t16_e64_:%[0-9]+]]:vgpr_16 = V_FMA_F16_gfx9_t16_e64 0, killed [[COPY]], 0, [[COPY1]], 0, [[V_MOV_B16_t16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
%0 = IMPLICIT_DEF
- %1 = COPY %0.sub0
- %2 = COPY %0.sub1
- %3 = V_MOV_B32_e32 1078523331, implicit $exec
- %4 = V_FMAC_F16_fake16_e64 0, killed %1, 0, %2, 0, %3, 0, 0, implicit $mode, implicit $exec
+ %1 = COPY %0.hi16
+ %2 = COPY %0.lo16
+ %3 = V_MOV_B16_t16_e64 0, 1078, 0, implicit $exec
+ %4 = V_FMAC_F16_t16_e64 0, killed %1, 0, %2, 0, %3, 0, 0, 0, implicit $mode, implicit $exec
...
---
name: test_fmaak_inline_literal_f16
tracksRegLiveness: true
liveins:
- - { reg: '$vgpr0', virtual-reg: '%0' }
+ - { reg: '$vgpr0_hi16', virtual-reg: '%0' }
body: |
bb.0:
liveins: $vgpr0
@@ -88,14 +88,13 @@ body: |
; GFX11-LABEL: name: test_fmaak_inline_literal_f16
; GFX11: liveins: $vgpr0
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
- ; GFX11-NEXT: [[V_FMA_F16_gfx9_fake16_e64_:%[0-9]+]]:vgpr_32 = V_FMA_F16_gfx9_fake16_e64 0, 16384, 0, killed [[COPY]], 0, [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY killed $vgpr0_hi16
+ ; GFX11-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 999, 0, implicit $exec
+ ; GFX11-NEXT: [[V_FMA_F16_gfx9_t16_e64_:%[0-9]+]]:vgpr_16 = V_FMA_F16_gfx9_t16_e64 0, 16384, 0, killed [[COPY]], 0, [[V_MOV_B16_t16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; GFX11-NEXT: S_ENDPGM 0
- %0:vgpr_32 = COPY killed $vgpr0
-
- %1:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
- %2:vgpr_32 = V_FMAC_F16_fake16_e64 0, 16384, 0, killed %0, 0, %1, 0, 0, implicit $mode, implicit $exec
+ %0:vgpr_16 = COPY killed $vgpr0_hi16
+ %1:vgpr_16 = V_MOV_B16_t16_e64 0, 999, 0, implicit $exec
+ %2:vgpr_16 = V_FMAC_F16_t16_e64 0, 16384, 0, killed %0, 0, %1, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
index 67acf16f70f3e..93b446e6dcbf6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
@@ -1,8 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.70 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1170 %s
-; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX12 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX12,GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX12,GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-FAKE16 %s
-; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-REAL16 %s
define amdgpu_cs float @test_cvt_f32_bf8_byte0(i32 %a) {
; GFX1170-LABEL: test_cvt_f32_bf8_byte0:
@@ -105,13 +106,31 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr
; GFX1170-NEXT: global_store_b32 v[3:4], v2, off
; GFX1170-NEXT: s_endpgm
;
-; GFX12-LABEL: test_cvt_pk_bf8_f32_word0:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1
-; GFX12-NEXT: global_store_b32 v[3:4], v2, off
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1
+; GFX12-TRUE16-NEXT: global_store_b32 v[3:4], v2, off
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_cvt_pk_bf8_f32_e64_dpp v2, v0, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX12-FAKE16-NEXT: global_store_b32 v[3:4], v2, off
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1
+; GFX1250-TRUE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0:
; GFX1250-FAKE16: ; %bb.0:
@@ -122,18 +141,6 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr
; GFX1250-FAKE16-NEXT: v_cvt_pk_bf8_f32_e64_dpp v2, v0, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: test_cvt_pk_bf8_f32_word0:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1
-; GFX1250-REAL16-NEXT: global_store_b32 v[4:5], v2, off
-; GFX1250-REAL16-NEXT: s_endpgm
%tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %a, i32 228, i32 15, i32 15, i1 1)
%tmp1 = bitcast i32 %tmp0 to float
%ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %tmp1, float %y, i32 %old, i1 false)
@@ -150,13 +157,33 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr
; GFX1170-NEXT: global_store_b32 v[3:4], v2, off
; GFX1170-NEXT: s_endpgm
;
-; GFX12-LABEL: test_cvt_pk_fp8_f32_word1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1
-; GFX12-NEXT: global_store_b32 v[3:4], v2, off
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1
+; GFX12-TRUE16-NEXT: global_store_b32 v[3:4], v2, off
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1]
+; GFX12-FAKE16-NEXT: global_store_b32 v[3:4], v2, off
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1
+; GFX1250-TRUE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1:
; GFX1250-FAKE16: ; %bb.0:
@@ -169,18 +196,6 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr
; GFX1250-FAKE16-NEXT: v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1]
; GFX1250-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: test_cvt_pk_fp8_f32_word1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1
-; GFX1250-REAL16-NEXT: global_store_b32 v[4:5], v2, off
-; GFX1250-REAL16-NEXT: s_endpgm
%tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %a, i32 228, i32 15, i32 15, i1 1)
%tmp1 = bitcast i32 %tmp0 to float
%ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %tmp1, float %y, i32 %old, i1 true)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 86f22fa12fa06..66bb9a9f5a838 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI-SDAG %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index 2ec91f678f85b..4876cd4c5f0a9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -1,8 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize32",+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize32",-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
; RUN: llc -mtriple=amdgpu10.10 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize32",+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize32",-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
declare i32 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index 4456597576b90..541a7c73e8c07 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -1,9 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
; RUN: llc -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GCN,VI,SDAG-VI %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,SDAG-GFX9 %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64",-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index 6837c3fcb293d..3c7b66bc14421 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -1,10 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgpu6.01 | FileCheck %s --check-prefixes=PREGFX10
-;RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgpu6.01 | FileCheck %s --check-prefixes=PREGFX10,GFX6
+;RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck %s --check-prefixes=PREGFX10,GFX8
;RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s --check-prefixes=GFX10
-;RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
-;RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-;RUN: llc < %s -global-isel -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+;RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 -mattr=+real-true16 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
+;RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 -mattr=-real-true16 | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
+;RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -mattr=+real-true16 | FileCheck %s --check-prefixes=GFX12,GFX12-TRUE16,GFX12-SDAG-TRUE16
+;RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -mattr=-real-true16 | FileCheck %s --check-prefixes=GFX12,GFX12-FAKE16,GFX12-SDAG-FAKE16
+;RUN: llc < %s -global-isel -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -mattr=+real-true16 | FileCheck %s --check-prefixes=GFX12,GFX12-TRUE16,GFX12-GISEL-TRUE16
+;RUN: llc < %s -global-isel -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -mattr=-real-true16 | FileCheck %s --check-prefixes=GFX12,GFX12-FAKE16,GFX12-GISEL-FAKE16
define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> inreg) {
; PREGFX10-LABEL: buffer_load:
@@ -435,6 +438,20 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) {
+; GFX6-LABEL: buffer_load_negative_offset:
+; GFX6: ; %bb.0: ; %main_body
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, -16, v0
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: buffer_load_negative_offset:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, -16, v0
+; GFX8-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_negative_offset:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0
@@ -462,6 +479,30 @@ main_body:
}
define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, ptr addrspace(3) %lds) {
+; GFX6-LABEL: buffer_load_mmo:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_write_b32 v0, v2
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 16, v0
+; GFX6-NEXT: ds_write_b32 v0, v2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: buffer_load_mmo:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: s_mov_b32 m0, -1
+; GFX8-NEXT: ds_write2_b32 v0, v2, v2 offset1:4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, v1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_mmo:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0
@@ -596,39 +637,73 @@ define amdgpu_ps void @buffer_load_x1_offen_merged_or(<4 x i32> inreg %rsrc, i32
; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
; GFX11-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v4, 6, v0
-; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
-; GFX12-SDAG-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX12-SDAG-NEXT: export mrt0, v0, v1, v2, v3 done
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: export mrt0, v4, v5, v0, v0 done
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, 6, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, 4, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, 8, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, 12, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v4, 16, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v5, 28, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, 32, v0
-; GFX12-GISEL-NEXT: s_clause 0x5
-; GFX12-GISEL-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v2, v2, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v3, v3, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v4, v4, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v5, v5, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x2
-; GFX12-GISEL-NEXT: export mrt0, v1, v2, v3, v4 done
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: export mrt0, v5, v0, v0, v0 done
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX12-SDAG-TRUE16-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-SDAG-TRUE16-NEXT: export mrt0, v0, v1, v2, v3 done
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: export mrt0, v4, v5, v0, v0 done
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-SDAG-FAKE16-NEXT: export mrt0, v0, v1, v2, v3 done
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: export mrt0, v4, v5, v0, v0 done
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 6, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v1, 4, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 8, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v3, 12, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 16, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 28, v0
+; GFX12-GISEL-TRUE16-NEXT: v_or_b32_e32 v0, 32, v0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x5
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v2, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v3, v3, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v4, v4, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v5, v5, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-GISEL-TRUE16-NEXT: export mrt0, v1, v2, v3, v4 done
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: export mrt0, v5, v0, v0, v0 done
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 6, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v1, 4, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 8, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v3, 12, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v4, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v5, 28, v0
+; GFX12-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, 32, v0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x5
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v2, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v3, v3, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v4, v4, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v5, v5, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-GISEL-FAKE16-NEXT: export mrt0, v1, v2, v3, v4 done
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: export mrt0, v5, v0, v0, v0 done
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
main_body:
%a = shl i32 %inp, 6
%a1 = or i32 %a, 4
@@ -921,6 +996,63 @@ define amdgpu_ps {<4 x float>, <2 x float>, float} @buffer_load_int(<4 x i32> in
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
+; GFX12-SDAG-TRUE16-LABEL: buffer_load_int:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x2
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-FAKE16-LABEL: buffer_load_int:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x2
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: buffer_load_int:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x2
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v5
+; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: buffer_load_int:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x2
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s4, v4
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v5
+; GFX12-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GISEL-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GISEL-FAKE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
; GFX12-SDAG-LABEL: buffer_load_int:
; GFX12-SDAG: ; %bb.0: ; %main_body
; GFX12-SDAG-NEXT: s_clause 0x2
@@ -929,7 +1061,6 @@ define amdgpu_ps {<4 x float>, <2 x float>, float} @buffer_load_int(<4 x i32> in
; GFX12-SDAG-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
; GFX12-GISEL-LABEL: buffer_load_int:
; GFX12-GISEL: ; %bb.0: ; %main_body
; GFX12-GISEL-NEXT: s_clause 0x2
@@ -1019,21 +1150,37 @@ define amdgpu_ps float @raw_buffer_load_i16(<4 x i32> inreg %rsrc) {
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: raw_buffer_load_i16:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: buffer_load_u16 v0, off, s[0:3], null
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: raw_buffer_load_i16:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: buffer_load_u16 v0, off, s[0:3], null
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-GISEL-NEXT: s_cvt_f32_u32 s0, s0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-SDAG-TRUE16-LABEL: raw_buffer_load_i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_u16 v0, off, s[0:3], null
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_buffer_load_i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_u16 v0, off, s[0:3], null
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_buffer_load_i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_u16 v0, off, s[0:3], null
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-TRUE16-NEXT: s_cvt_f32_u32 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_buffer_load_i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_u16 v0, off, s[0:3], null
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-FAKE16-NEXT: s_cvt_f32_u32 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
main_body:
%tmp = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
%tmp2 = zext i16 %tmp to i32
@@ -1063,21 +1210,37 @@ define amdgpu_ps float @raw_buffer_load_sbyte(<4 x i32> inreg %rsrc) {
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: raw_buffer_load_sbyte:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: buffer_load_i8 v0, off, s[0:3], null
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: raw_buffer_load_sbyte:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: buffer_load_i8 v0, off, s[0:3], null
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-GISEL-NEXT: s_cvt_f32_i32 s0, s0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-SDAG-TRUE16-LABEL: raw_buffer_load_sbyte:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_i8 v0, off, s[0:3], null
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_buffer_load_sbyte:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_i8 v0, off, s[0:3], null
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_buffer_load_sbyte:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_i8 v0, off, s[0:3], null
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-TRUE16-NEXT: s_cvt_f32_i32 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_buffer_load_sbyte:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_i8 v0, off, s[0:3], null
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-FAKE16-NEXT: s_cvt_f32_i32 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
main_body:
%tmp = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
%tmp2 = sext i8 %tmp to i32
@@ -1107,21 +1270,37 @@ define amdgpu_ps float @raw_buffer_load_sshort(<4 x i32> inreg %rsrc) {
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: raw_buffer_load_sshort:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: buffer_load_i16 v0, off, s[0:3], null
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: raw_buffer_load_sshort:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: buffer_load_i16 v0, off, s[0:3], null
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-GISEL-NEXT: s_cvt_f32_i32 s0, s0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-SDAG-TRUE16-LABEL: raw_buffer_load_sshort:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_i16 v0, off, s[0:3], null
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_buffer_load_sshort:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_i16 v0, off, s[0:3], null
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_buffer_load_sshort:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_i16 v0, off, s[0:3], null
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-TRUE16-NEXT: s_cvt_f32_i32 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_buffer_load_sshort:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_i16 v0, off, s[0:3], null
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-FAKE16-NEXT: s_cvt_f32_i32 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
main_body:
%tmp = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
%tmp2 = sext i16 %tmp to i32
@@ -1152,21 +1331,37 @@ define amdgpu_ps void @raw_buffer_load_f16(<4 x i32> inreg %rsrc, ptr addrspace(
; GFX11-NEXT: ds_store_b16 v0, v1
; GFX11-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: raw_buffer_load_f16:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: buffer_load_u16 v1, off, s[0:3], null
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: ds_store_b16 v0, v1
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: raw_buffer_load_f16:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: buffer_load_u16 v1, off, s[0:3], null
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX12-GISEL-NEXT: ds_store_b16 v0, v1
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX12-SDAG-TRUE16-LABEL: raw_buffer_load_f16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_u16 v1, off, s[0:3], null
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: ds_store_b16 v0, v1
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_buffer_load_f16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %main_body
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_u16 v1, off, s[0:3], null
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: ds_store_b16 v0, v1
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_buffer_load_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_u16 v1, off, s[0:3], null
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ds_store_b16 v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_buffer_load_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_u16 v1, off, s[0:3], null
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-GISEL-FAKE16-NEXT: ds_store_b16 v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
main_body:
%val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
store half %val, ptr addrspace(3) %ptr
@@ -1509,3 +1704,8 @@ declare half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32>, i32, i32, i32) #0
declare <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32>, i32, i32, i32) #0
declare <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32>, i32, i32, i32) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
+; GFX12-FAKE16: {{.*}}
+; GFX12-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll
index c9d2623aca41c..6275520f601f9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
declare half @llvm.floor.f16(half %a)
declare <2 x half> @llvm.floor.v2f16(<2 x half> %a)
@@ -46,23 +46,23 @@ define amdgpu_kernel void @floor_f16(
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
-; GFX11-LABEL: floor_f16:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s10, s6
-; GFX11-NEXT: s_mov_b32 s11, s7
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s8, s2
-; GFX11-NEXT: s_mov_b32 s9, s3
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_floor_f16_e32 v0.l, v0.l
-; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: floor_f16:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
+; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: floor_f16:
; GFX11-FAKE16: ; %bb.0: ; %entry
@@ -145,26 +145,26 @@ define amdgpu_kernel void @floor_v2f16(
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
-; GFX11-LABEL: floor_v2f16:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-NEXT: s_mov_b32 s10, s6
-; GFX11-NEXT: s_mov_b32 s11, s7
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s8, s2
-; GFX11-NEXT: s_mov_b32 s9, s3
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_floor_f16_e32 v0.l, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_floor_f16_e32 v0.h, v1.l
-; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: floor_v2f16:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
+; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
+; GFX11-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: floor_v2f16:
; GFX11-FAKE16: ; %bb.0: ; %entry
@@ -195,3 +195,5 @@ entry:
store <2 x half> %r.val, ptr addrspace(1) %r
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
index baefe561e94ae..865cc0e61a919 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
@@ -1071,6 +1071,214 @@ define amdgpu_ps double @test_ldexp_f64_i32_uniform(double inreg %a, i32 inreg %
ret double %result
}
+define half @test_ldexp_f16_i32_neg(ptr addrspace(1) %out, half %a, i32 %b) {
+; GFX6-LABEL: test_ldexp_f16_i32_neg:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, 0, v3
+; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: test_ldexp_f16_i32_neg:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 0, v3
+; GFX8-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX8-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: test_ldexp_f16_i32_neg:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v3
+; GFX9-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX9-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: test_ldexp_f16_i32_neg:
+; GFX11-SDAG-TRUE16: ; %bb.0:
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_nc_u32_e32 v0, 0, v3
+; GFX11-SDAG-TRUE16-NEXT: s_movk_i32 s0, 0x8000
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-FAKE16-LABEL: test_ldexp_f16_i32_neg:
+; GFX11-SDAG-FAKE16: ; %bb.0:
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_sub_nc_u32_e32 v0, 0, v3
+; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x8000
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v0, v0, s0, 0x7fff
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-TRUE16-LABEL: test_ldexp_f16_i32_neg:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_sub_nc_u32_e32 v0, 0, v3
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_med3_i32 v0, 0xffff8000, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_ldexp_f16_i32_neg:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_sub_nc_u32_e32 v0, 0, v3
+; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, v0, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %b.neg = sub i32 0, %b
+ %result = call half @llvm.ldexp.f16_i32(half %a, i32 %b.neg)
+ ret half %result
+}
+
+define half @ldexp_f16_i32_imm_a(ptr addrspace(1) %out, i32 %b) {
+; GFX6-LABEL: ldexp_f16_i32_imm_a:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_ldexp_f32_e32 v0, 2.0, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: ldexp_f16_i32_imm_a:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v0, v2, s4, v0
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: ldexp_f16_i32_imm_a:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v0, v2, s4, v0
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: ldexp_f16_i32_imm_a:
+; GFX11-SDAG-TRUE16: ; %bb.0:
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: s_movk_i32 s0, 0x8000
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v0, v2, s0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, 2.0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-FAKE16-LABEL: ldexp_f16_i32_imm_a:
+; GFX11-SDAG-FAKE16: ; %bb.0:
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x8000
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v0, v2, s0, 0x7fff
+; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: ldexp_f16_i32_imm_a:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v0, v2, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: ldexp_f16_i32_imm_a:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-TRUE16-LABEL: ldexp_f16_i32_imm_a:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_med3_i32 v0, 0xffff8000, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, 2.0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: ldexp_f16_i32_imm_a:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, 2.0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %result = call half @llvm.ldexp.f16_i32(half 2.0, i32 %b)
+ ret half %result
+}
+
+define half @test_ldexp_f16_i32_imm_b(ptr addrspace(1) %out, half %a) {
+; GFX6-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GFX6-NEXT: v_ldexp_f32_e64 v0, v0, 2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_ldexp_f16_e64 v0, v2, 2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_ldexp_f16_e64 v0, v2, 2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX11-SDAG-TRUE16: ; %bb.0:
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_ldexp_f16_e64 v0.l, v2.l, 2
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-FAKE16-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX11-SDAG-FAKE16: ; %bb.0:
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e64 v0, v2, 2
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-TRUE16-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_ldexp_f16_e64 v0.l, v2.l, 2
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_ldexp_f16_i32_imm_b:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e64 v0, v2, 2
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %result = call half @llvm.ldexp.f16_i32(half %a, i32 2)
+ ret half %result
+}
+
declare float @llvm.ldexp.f32.i32(float, i32) #0
declare float @llvm.ldexp.f32.i16(float, i16) #0
declare float @llvm.ldexp.f32.i64(float, i64) #0
@@ -1091,7 +1299,3 @@ attributes #0 = { nounwind readnone }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX6-GISEL: {{.*}}
; GFX6-SDAG: {{.*}}
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index c5106d4fc0d66..7355ab59e1bbd 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -2,9 +2,10 @@
; RUN: llc -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -mtriple=amdgpu8.02-- < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
-; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
-; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
define amdgpu_kernel void @constant_load_i1(ptr addrspace(1) %out, ptr addrspace(4) nocapture %in) #0 {
; GFX6-LABEL: constant_load_i1:
@@ -153,15 +154,38 @@ define amdgpu_kernel void @constant_load_v2i1(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_load_v2i1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_load_v2i1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_load_v2i1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v0, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_load_v2i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_load_v2i1:
; GFX1250-FAKE16: ; %bb.0:
@@ -175,19 +199,6 @@ define amdgpu_kernel void @constant_load_v2i1(ptr addrspace(1) %out, ptr addrspa
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_load_v2i1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
store <2 x i1> %load, ptr addrspace(1) %out
ret void
@@ -249,15 +260,38 @@ define amdgpu_kernel void @constant_load_v3i1(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_load_v3i1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_load_v3i1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_load_v3i1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v0, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_load_v3i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_load_v3i1:
; GFX1250-FAKE16: ; %bb.0:
@@ -271,19 +305,6 @@ define amdgpu_kernel void @constant_load_v3i1(ptr addrspace(1) %out, ptr addrspa
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_load_v3i1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
store <3 x i1> %load, ptr addrspace(1) %out
ret void
@@ -346,15 +367,38 @@ define amdgpu_kernel void @constant_load_v4i1(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_load_v4i1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_load_v4i1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_load_v4i1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v0, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_load_v4i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_load_v4i1:
; GFX1250-FAKE16: ; %bb.0:
@@ -368,19 +412,6 @@ define amdgpu_kernel void @constant_load_v4i1(ptr addrspace(1) %out, ptr addrspa
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_load_v4i1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
store <4 x i1> %load, ptr addrspace(1) %out
ret void
@@ -443,15 +474,38 @@ define amdgpu_kernel void @constant_load_v8i1(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_load_v8i1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_load_v8i1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_load_v8i1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v0, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_load_v8i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_load_v8i1:
; GFX1250-FAKE16: ; %bb.0:
@@ -465,19 +519,6 @@ define amdgpu_kernel void @constant_load_v8i1(ptr addrspace(1) %out, ptr addrspa
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_load_v8i1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: global_store_b8 v1, v0, s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <8 x i1>, ptr addrspace(4) %in
store <8 x i1> %load, ptr addrspace(1) %out
ret void
@@ -540,15 +581,38 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_load_v16i1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_load_v16i1:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_load_v16i1:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_load_v16i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_load_v16i1:
; GFX1250-FAKE16: ; %bb.0:
@@ -562,19 +626,6 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_load_v16i1:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u16 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
store <16 x i1> %load, ptr addrspace(1) %out
ret void
@@ -1088,19 +1139,33 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i32(ptr addrspace(1) %out
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v2i1_to_v2i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v2, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v2i1_to_v2i32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v2, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v2i1_to_v2i32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v2, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_zextload_v2i1_to_v2i32:
; GFX1250: ; %bb.0:
@@ -1273,21 +1338,58 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
; EG-NEXT: ADD_INT * T3.X, PV.X, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v3i1_to_v3i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v1, v3, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 2, v0
-; GFX12-NEXT: v_and_b32_e32 v0, 1, v1
-; GFX12-NEXT: v_bfe_u32 v1, v1, 1, 1
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: global_store_b96 v3, v[0:2], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v3i1_to_v3i32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v1, v3, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v1, v1, 1, 1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v3i1_to_v3i32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v3, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v1, v1, 1, 1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v3i1_to_v3i32:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v3, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s2, 1
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-TRUE16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v3i1_to_v3i32:
; GFX1250-FAKE16: ; %bb.0:
@@ -1306,27 +1408,6 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-FAKE16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v3i1_to_v3i32:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v3, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
-; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 2
-; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
-; GFX1250-REAL16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
%ext = zext <3 x i1> %load to <3 x i32>
store <3 x i32> %ext, ptr addrspace(1) %out
@@ -1493,21 +1574,58 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v4i1_to_v4i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v1, v4, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX12-NEXT: v_and_b32_e32 v0, 1, v1
-; GFX12-NEXT: v_bfe_u32 v2, v1, 2, 1
-; GFX12-NEXT: v_bfe_u32 v1, v1, 1, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v4i1_to_v4i32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v1, v4, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v1, 2, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v1, v1, 1, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v4i1_to_v4i32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v1, v4, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v1, 2, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v1, v1, 1, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v4i1_to_v4i32:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v4, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s2, 1
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10002
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s3, 3
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX1250-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v4i1_to_v4i32:
; GFX1250-FAKE16: ; %bb.0:
@@ -1527,27 +1645,6 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 3, v3
; GFX1250-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v4i1_to_v4i32:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v4, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10002
-; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
-; GFX1250-REAL16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
%ext = zext <4 x i1> %load to <4 x i32>
store <4 x i32> %ext, ptr addrspace(1) %out
@@ -1750,32 +1847,88 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; EG-NEXT: ADD_INT * T8.X, PS, literal.x,
; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v8i1_to_v8i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v8, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10005
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
-; GFX12-NEXT: s_and_b32 s6, s2, 1
-; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10002
-; GFX12-NEXT: s_bfe_u32 s2, s2, 0x10004
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 7, v0
-; GFX12-NEXT: v_bfe_u32 v2, v0, 6, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s4
-; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, s7
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v8, v[4:7], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v8i1_to_v8i32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v8, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10005
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s2, 1
+; GFX12-TRUE16-NEXT: s_bfe_u32 s7, s2, 0x10002
+; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10004
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 7, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 6, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, s7
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v8i1_to_v8i32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v8, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10005
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-FAKE16-NEXT: s_and_b32 s6, s2, 1
+; GFX12-FAKE16-NEXT: s_bfe_u32 s7, s2, 0x10002
+; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s2, 0x10004
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 7, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 6, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, s7
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v8i1_to_v8i32:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v8, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s5, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s6, s2, 0x10005
+; GFX1250-TRUE16-NEXT: s_and_b32 s7, s2, 1
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s8, s2, 0x10002
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10004
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s9, s5, 7
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s5, 0x10006
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v2, s5
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v7, s3
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v5, s4
+; GFX1250-TRUE16-NEXT: s_clause 0x1
+; GFX1250-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX1250-TRUE16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v8i1_to_v8i32:
; GFX1250-FAKE16: ; %bb.0:
@@ -1804,35 +1957,6 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1250-FAKE16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v8i1_to_v8i32:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v8, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v8, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s5, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10001
-; GFX1250-REAL16-NEXT: s_bfe_u32 s6, s2, 0x10005
-; GFX1250-REAL16-NEXT: s_and_b32 s7, s2, 1
-; GFX1250-REAL16-NEXT: s_bfe_u32 s8, s2, 0x10002
-; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10004
-; GFX1250-REAL16-NEXT: s_lshr_b32 s9, s5, 7
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s5, 0x10006
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v2, s5
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v7, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v5, s4
-; GFX1250-REAL16-NEXT: s_clause 0x1
-; GFX1250-REAL16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
-; GFX1250-REAL16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <8 x i1>, ptr addrspace(4) %in
%ext = zext <8 x i1> %load to <8 x i32>
store <8 x i32> %ext, ptr addrspace(1) %out
@@ -2134,46 +2258,86 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T14.X, T11.X, literal.x,
; EG-NEXT: 12(1.681558e-44), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v16i1_to_v16i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_b16 v0, v16, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10003
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10001
-; GFX12-NEXT: s_bfe_u32 s6, s2, 0x10007
-; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-NEXT: s_and_b32 s9, s2, 1
-; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-NEXT: s_bfe_u32 s11, s3, 0x10005
-; GFX12-NEXT: s_bfe_u32 s12, s3, 0x1000b
-; GFX12-NEXT: s_lshr_b32 s13, s3, 15
-; GFX12-NEXT: s_bfe_u32 s14, s3, 0x10002
-; GFX12-NEXT: s_bfe_u32 s15, s3, 0x10006
-; GFX12-NEXT: s_bfe_u32 s16, s3, 0x10004
-; GFX12-NEXT: s_bfe_u32 s17, s3, 0x10008
-; GFX12-NEXT: s_bfe_u32 s3, s3, 0x1000e
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX12-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v2, s3
-; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
-; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX12-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v4, s17
-; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
-; GFX12-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s9
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i32:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v16, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10003
+; GFX12-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10001
+; GFX12-TRUE16-NEXT: s_bfe_u32 s6, s2, 0x10007
+; GFX12-TRUE16-NEXT: s_bfe_u32 s7, s2, 0x10009
+; GFX12-TRUE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s2, 1
+; GFX12-TRUE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
+; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
+; GFX12-TRUE16-NEXT: s_bfe_u32 s11, s3, 0x10005
+; GFX12-TRUE16-NEXT: s_bfe_u32 s12, s3, 0x1000b
+; GFX12-TRUE16-NEXT: s_lshr_b32 s13, s3, 15
+; GFX12-TRUE16-NEXT: s_bfe_u32 s14, s3, 0x10002
+; GFX12-TRUE16-NEXT: s_bfe_u32 s15, s3, 0x10006
+; GFX12-TRUE16-NEXT: s_bfe_u32 s16, s3, 0x10004
+; GFX12-TRUE16-NEXT: s_bfe_u32 s17, s3, 0x10008
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x1000e
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v2, s3
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v4, s17
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s9
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
+; GFX12-TRUE16-NEXT: s_clause 0x3
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i32:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u16 v0, v16, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-FAKE16-NEXT: s_and_b32 s6, 0xffff, s2
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10007
+; GFX12-FAKE16-NEXT: s_bfe_u32 s7, s2, 0x10009
+; GFX12-FAKE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
+; GFX12-FAKE16-NEXT: s_and_b32 s9, s2, 1
+; GFX12-FAKE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
+; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
+; GFX12-FAKE16-NEXT: s_bfe_u32 s11, s6, 0x10005
+; GFX12-FAKE16-NEXT: s_bfe_u32 s12, s6, 0x1000b
+; GFX12-FAKE16-NEXT: s_lshr_b32 s13, s6, 15
+; GFX12-FAKE16-NEXT: s_bfe_u32 s14, s6, 0x10002
+; GFX12-FAKE16-NEXT: s_bfe_u32 s15, s6, 0x10006
+; GFX12-FAKE16-NEXT: s_bfe_u32 s16, s6, 0x10004
+; GFX12-FAKE16-NEXT: s_bfe_u32 s17, s6, 0x10008
+; GFX12-FAKE16-NEXT: s_bfe_u32 s6, s6, 0x1000e
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v2, s6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v4, s17
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
+; GFX12-FAKE16-NEXT: s_clause 0x3
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_zextload_v16i1_to_v16i32:
; GFX1250: ; %bb.0:
@@ -5419,24 +5583,66 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v2i1_to_v2i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v2i1_to_v2i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v2i1_to_v2i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v2i1_to_v2i64:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v2i1_to_v2i64:
; GFX1250-FAKE16: ; %bb.0:
@@ -5457,29 +5663,6 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v2i1_to_v2i64:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
-; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 1
-; GFX1250-REAL16-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
-; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
%ext = zext <2 x i1> %load to <2 x i64>
store <2 x i64> %ext, ptr addrspace(1) %out
@@ -5548,22 +5731,39 @@ define amdgpu_kernel void @constant_sextload_v2i1_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: MOV * T1.W, T1.Z,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_sextload_v2i1_to_v2i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v4, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v1, 1, v0
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_bfe_i32 v2, v1, 0, 1
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_sextload_v2i1_to_v2i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v4, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v0
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 1
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_sextload_v2i1_to_v2i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v4, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 1, v0
+; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 1
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v2i1_to_v2i64:
; GFX1250: ; %bb.0:
@@ -5665,26 +5865,76 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; EG-NEXT: ADD_INT * T3.X, PV.X, literal.x,
; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v3i1_to_v3i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v5, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-NEXT: v_bfe_u32 v2, v0, 1, 1
-; GFX12-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, 2, v1
-; GFX12-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v2, 0xffff, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v5, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v3i1_to_v3i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v5, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 1, 1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v0, 1, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v3i1_to_v3i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v5, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 1, 1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v0, 1, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v3i1_to_v3i64:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s3
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s4
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-TRUE16-NEXT: global_store_b64 v1, v[0:1], s[0:1] offset:16
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v3i1_to_v3i64:
; GFX1250-FAKE16: ; %bb.0:
@@ -5711,35 +5961,6 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
; GFX1250-FAKE16-NEXT: global_store_b128 v5, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v3i1_to_v3i64:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10001
-; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 2
-; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s3
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s4
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
-; GFX1250-REAL16-NEXT: global_store_b64 v1, v[0:1], s[0:1] offset:16
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
%ext = zext <3 x i1> %load to <3 x i64>
store <3 x i64> %ext, ptr addrspace(1) %out
@@ -5826,28 +6047,51 @@ define amdgpu_kernel void @constant_sextload_v3i1_to_v3i64(ptr addrspace(1) %out
; EG-NEXT: MOV * T1.W, T1.Z,
; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_sextload_v3i1_to_v3i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v6, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v6, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v1, 2, v0
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 1, v0
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_i32 v4, v1, 0, 1
-; GFX12-NEXT: v_bfe_i32 v2, v2, 0, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v6, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_sextload_v3i1_to_v3i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v6, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v0
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_i32 v4, v1, 0, 1
+; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_sextload_v3i1_to_v3i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v6, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 1, v0
+; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_i32 v4, v1, 0, 1
+; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v3i1_to_v3i64:
; GFX1250: ; %bb.0:
@@ -5959,35 +6203,93 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; EG-NEXT: ADD_INT * T3.X, PV.X, literal.x,
; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v4i1_to_v4i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10002
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 3, v0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_and_b32 s2, s2, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v4i1_to_v4i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10002
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 3, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v4i1_to_v4i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10002
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 3, v0
+; GFX12-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-FAKE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v4i1_to_v4i64:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s3, 3
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX1250-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v4i1_to_v4i64:
; GFX1250-FAKE16: ; %bb.0:
@@ -6019,34 +6321,6 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v4i1_to_v4i64:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
-; GFX1250-REAL16-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
-; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
-; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
%ext = zext <4 x i1> %load to <4 x i64>
store <4 x i64> %ext, ptr addrspace(1) %out
@@ -6143,32 +6417,59 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i64(ptr addrspace(1) %out
; EG-NEXT: MOV * T2.W, T2.Z,
; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_sextload_v4i1_to_v4i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v8, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v1, 3, v0
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 2, v0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 1, v0
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_bfe_i32 v6, v1, 0, 1
-; GFX12-NEXT: v_bfe_i32 v4, v2, 0, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_bfe_i32 v2, v3, 0, 1
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX12-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_sextload_v4i1_to_v4i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v8, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_i32 v6, v1, 0, 1
+; GFX12-TRUE16-NEXT: v_bfe_i32 v4, v2, 0, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 1
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_sextload_v4i1_to_v4i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v8, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 3, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 1
+; GFX12-FAKE16-NEXT: v_bfe_i32 v4, v2, 0, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v3, 0, 1
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v4i1_to_v4i64:
; GFX1250: ; %bb.0:
@@ -6327,35 +6628,65 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i64(ptr addrspace(1) %out
; EG-NEXT: ADD_INT * T12.X, T9.X, literal.x,
; EG-NEXT: 12(1.681558e-44), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v8i1_to_v8i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v7, v1
-; GFX12-NEXT: v_mov_b32_e32 v9, v1
-; GFX12-NEXT: v_mov_b32_e32 v11, v1
-; GFX12-NEXT: v_mov_b32_e32 v13, v1
-; GFX12-NEXT: v_mov_b32_e32 v15, v1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v12, v1, s[2:3]
-; GFX12-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_and_b32 v0, 0xffff, v12
-; GFX12-NEXT: v_bfe_u32 v6, v12, 5, 1
-; GFX12-NEXT: v_bfe_u32 v4, v12, 4, 1
-; GFX12-NEXT: v_bfe_u32 v10, v12, 3, 1
-; GFX12-NEXT: v_bfe_u32 v8, v12, 2, 1
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 7, v0
-; GFX12-NEXT: v_bfe_u32 v0, v0, 6, 1
-; GFX12-NEXT: v_bfe_u32 v14, v12, 1, 1
-; GFX12-NEXT: v_and_b32_e32 v12, 1, v12
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v1, v[12:15], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v8i1_to_v8i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v11, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v13, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v15, v1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v12, v1, s[2:3]
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_and_b32 v0, 0xffff, v12
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v12, 5, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v12, 4, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v12, 3, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v12, 2, 1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 7, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v0, 6, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v14, v12, 1, 1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 1, v12
+; GFX12-TRUE16-NEXT: s_clause 0x3
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:32
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v1, v[12:15], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v8i1_to_v8i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v11, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v13, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v15, v1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v12, v1, s[2:3]
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_and_b32 v0, 0xffff, v12
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v12, 5, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v12, 4, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v12, 3, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v12, 2, 1
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 7, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v0, 6, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v14, v12, 1, 1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v12, 1, v12
+; GFX12-FAKE16-NEXT: s_clause 0x3
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:32
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v1, v[12:15], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_zextload_v8i1_to_v8i64:
; GFX1250: ; %bb.0:
@@ -6549,48 +6880,91 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; EG-NEXT: MOV * T8.W, T8.Z,
; EG-NEXT: 12(1.681558e-44), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_sextload_v8i1_to_v8i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_u8 v0, v16, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v9, s3
-; GFX12-NEXT: s_lshr_b32 s2, s3, 6
-; GFX12-NEXT: s_lshr_b32 s4, s3, 7
-; GFX12-NEXT: s_lshr_b32 s6, s3, 4
-; GFX12-NEXT: s_lshr_b32 s8, s3, 5
-; GFX12-NEXT: s_lshr_b32 s10, s3, 2
-; GFX12-NEXT: s_lshr_b32 s12, s3, 3
-; GFX12-NEXT: s_lshr_b32 s14, s3, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: v_bfe_i32 v12, v9, 0, 1
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
-; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
-; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
-; GFX12-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX12-NEXT: v_mov_b32_e32 v15, s15
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_sextload_v8i1_to_v8i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_u8 v0, v16, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s3, 6
+; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s3, 7
+; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s3, 4
+; GFX12-TRUE16-NEXT: s_lshr_b32 s8, s3, 5
+; GFX12-TRUE16-NEXT: s_lshr_b32 s10, s3, 2
+; GFX12-TRUE16-NEXT: s_lshr_b32 s12, s3, 3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s14, s3, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-TRUE16-NEXT: v_bfe_i32 v12, v9, 0, 1
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v15, s15
+; GFX12-TRUE16-NEXT: s_clause 0x3
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_sextload_v8i1_to_v8i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u8 v0, v16, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, s3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s3, 6
+; GFX12-FAKE16-NEXT: s_lshr_b32 s4, s3, 7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s3, 4
+; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s3, 5
+; GFX12-FAKE16-NEXT: s_lshr_b32 s10, s3, 2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s3, 3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s14, s3, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-FAKE16-NEXT: v_bfe_i32 v12, v9, 0, 1
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v15, s15
+; GFX12-FAKE16-NEXT: s_clause 0x3
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v8i1_to_v8i64:
; GFX1250: ; %bb.0:
@@ -6856,65 +7230,185 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T22.X, T15.X, literal.x,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-NEXT: v_mov_b32_e32 v9, v3
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_b16 v0, v3, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-NEXT: s_and_b32 s2, s2, 1
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v3
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v6, 11, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 8, 1
+; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10006
+; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 5, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v6, 14, 1
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10004
+; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v0
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
+; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v3
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u16 v0, v3, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v6, 11, 1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 8, 1
+; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10006
+; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 5, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v6, 14, 1
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10004
+; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v0
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
+; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: global_load_u16 v0, v3, s[2:3] nv
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000b
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s5
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10009
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
; GFX1250-FAKE16: ; %bb.0:
@@ -6961,66 +7455,6 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[24:27], s[0:1] offset:16
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[28:31], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: global_wb
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u16 v0, v3, s[2:3] nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[0:3], s[0:1]
-; GFX1250-REAL16-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
store <16 x i64> %ext, ptr addrspace(1) %out
@@ -7303,77 +7737,149 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: MOV * T14.W, T14.Z,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
;
-; GFX12-LABEL: constant_sextload_v16i1_to_v16i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v32, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_b16 v0, v32, s[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v28, s3
-; GFX12-NEXT: s_lshr_b32 s2, s3, 14
-; GFX12-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-NEXT: s_lshr_b32 s8, s3, 13
-; GFX12-NEXT: s_lshr_b32 s10, s3, 10
-; GFX12-NEXT: s_lshr_b32 s12, s3, 11
-; GFX12-NEXT: s_lshr_b32 s6, s3, 12
-; GFX12-NEXT: s_lshr_b32 s14, s3, 8
-; GFX12-NEXT: s_lshr_b32 s16, s3, 9
-; GFX12-NEXT: s_lshr_b32 s18, s3, 6
-; GFX12-NEXT: s_lshr_b32 s20, s3, 7
-; GFX12-NEXT: s_lshr_b32 s22, s3, 4
-; GFX12-NEXT: s_lshr_b32 s24, s3, 5
-; GFX12-NEXT: s_lshr_b32 s26, s3, 2
-; GFX12-NEXT: s_lshr_b32 s28, s3, 3
-; GFX12-NEXT: s_lshr_b32 s30, s3, 1
-; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
-; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
-; GFX12-NEXT: v_mov_b32_e32 v13, s15
-; GFX12-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
-; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
-; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
-; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
-; GFX12-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
-; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
-; GFX12-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
-; GFX12-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
-; GFX12-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
-; GFX12-NEXT: v_mov_b32_e32 v30, s30
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-NEXT: s_clause 0x5
-; GFX12-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v32, v[28:31], s[0:1]
-; GFX12-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: constant_sextload_v16i1_to_v16i64:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v32, s[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v28, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s3, 14
+; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-TRUE16-NEXT: s_lshr_b32 s8, s3, 13
+; GFX12-TRUE16-NEXT: s_lshr_b32 s10, s3, 10
+; GFX12-TRUE16-NEXT: s_lshr_b32 s12, s3, 11
+; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s3, 12
+; GFX12-TRUE16-NEXT: s_lshr_b32 s14, s3, 8
+; GFX12-TRUE16-NEXT: s_lshr_b32 s16, s3, 9
+; GFX12-TRUE16-NEXT: s_lshr_b32 s18, s3, 6
+; GFX12-TRUE16-NEXT: s_lshr_b32 s20, s3, 7
+; GFX12-TRUE16-NEXT: s_lshr_b32 s22, s3, 4
+; GFX12-TRUE16-NEXT: s_lshr_b32 s24, s3, 5
+; GFX12-TRUE16-NEXT: s_lshr_b32 s26, s3, 2
+; GFX12-TRUE16-NEXT: s_lshr_b32 s28, s3, 3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s30, s3, 1
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v13, s15
+; GFX12-TRUE16-NEXT: v_bfe_i32 v28, v28, 0, 1
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
+; GFX12-TRUE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v30, s30
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
+; GFX12-TRUE16-NEXT: s_clause 0x5
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
+; GFX12-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: constant_sextload_v16i1_to_v16i64:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u16 v0, v32, s[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v28, s3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s3, 14
+; GFX12-FAKE16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s3, 13
+; GFX12-FAKE16-NEXT: s_lshr_b32 s10, s3, 10
+; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s3, 11
+; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s3, 12
+; GFX12-FAKE16-NEXT: s_lshr_b32 s14, s3, 8
+; GFX12-FAKE16-NEXT: s_lshr_b32 s16, s3, 9
+; GFX12-FAKE16-NEXT: s_lshr_b32 s18, s3, 6
+; GFX12-FAKE16-NEXT: s_lshr_b32 s20, s3, 7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s22, s3, 4
+; GFX12-FAKE16-NEXT: s_lshr_b32 s24, s3, 5
+; GFX12-FAKE16-NEXT: s_lshr_b32 s26, s3, 2
+; GFX12-FAKE16-NEXT: s_lshr_b32 s28, s3, 3
+; GFX12-FAKE16-NEXT: s_lshr_b32 s30, s3, 1
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v13, s15
+; GFX12-FAKE16-NEXT: v_bfe_i32 v28, v28, 0, 1
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
+; GFX12-FAKE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v30, s30
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
+; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
+; GFX12-FAKE16-NEXT: s_clause 0x5
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
+; GFX12-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index 0f6a9317aa9d6..0b3d591358028 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -3,8 +3,10 @@
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu9.00 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX9 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu10.10 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.50 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX1250 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu13.10 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX13 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG %s
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index aa80c946124a4..0bfa9df1f45c9 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GCN-GISEL %s
; TODO: GlobalISel should avoid generating v_ldexp_f32.
@@ -183,16 +185,49 @@ define amdgpu_cs half @v_s_amdgcn_log_f16(half inreg %src) {
}
define void @v_amdgcn_log_f16_div(half %src, ptr addrspace(1) %out) {
-; GFX12-LABEL: v_amdgcn_log_f16_div:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_log_f16_e32 v0.l, v0.l
-; GFX12-NEXT: global_store_b16 v[1:2], v0, off
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-TRUE16-LABEL: v_amdgcn_log_f16_div:
+; GFX12-SDAG-TRUE16: ; %bb.0:
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_log_f16_e32 v0.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-FAKE16-LABEL: v_amdgcn_log_f16_div:
+; GFX12-SDAG-FAKE16: ; %bb.0:
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-TRUE16-LABEL: v_amdgcn_log_f16_div:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_log_f16_e32 v0.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: v_amdgcn_log_f16_div:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX12-GISEL-FAKE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GCN-GISEL-LABEL: v_amdgcn_log_f16_div:
; GCN-GISEL: ; %bb.0:
@@ -265,16 +300,49 @@ define amdgpu_cs half @v_s_rcp_f16(half inreg %src) {
}
define void @v_rcp_f16_div(half %src, ptr addrspace(1) %out) {
-; GFX12-LABEL: v_rcp_f16_div:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_rcp_f16_e32 v0.l, v0.l
-; GFX12-NEXT: global_store_b16 v[1:2], v0, off
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-TRUE16-LABEL: v_rcp_f16_div:
+; GFX12-SDAG-TRUE16: ; %bb.0:
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-FAKE16-LABEL: v_rcp_f16_div:
+; GFX12-SDAG-FAKE16: ; %bb.0:
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-TRUE16-LABEL: v_rcp_f16_div:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: v_rcp_f16_div:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
+; GFX12-GISEL-FAKE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GCN-GISEL-LABEL: v_rcp_f16_div:
; GCN-GISEL: ; %bb.0:
@@ -684,13 +752,37 @@ define amdgpu_cs float @fdiv_f32_i32(float inreg %a, i32 inreg %b) {
}
define amdgpu_cs half @fdiv_f16_i16(half inreg %a, i16 inreg %b) {
-; GFX12-LABEL: fdiv_f16_i16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_cvt_f16_u16_e32 v0.l, s1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX12-NEXT: v_rcp_f16_e32 v0.l, v0.l
-; GFX12-NEXT: v_mul_f16_e32 v0.l, s0, v0.l
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-TRUE16-LABEL: fdiv_f16_i16:
+; GFX12-SDAG-TRUE16: ; %bb.0:
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.l, s1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, s0, v0.l
+; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-FAKE16-LABEL: fdiv_f16_i16:
+; GFX12-SDAG-FAKE16: ; %bb.0:
+; GFX12-SDAG-FAKE16-NEXT: v_cvt_f16_u16_e32 v0, s1
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
+; GFX12-SDAG-FAKE16-NEXT: v_mul_f16_e32 v0, s0, v0
+; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-TRUE16-LABEL: fdiv_f16_i16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.l, s1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, s0, v0.l
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: fdiv_f16_i16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_cvt_f16_u16_e32 v0, s1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, s0, v0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: fdiv_f16_i16:
; GCN-GISEL: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/rotl.ll b/llvm/test/CodeGen/AMDGPU/rotl.ll
index c74d485767922..0923f1977db0d 100644
--- a/llvm/test/CodeGen/AMDGPU/rotl.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotl.ll
@@ -3,8 +3,10 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG,GFX11-SDAG-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG,GFX11-SDAG-FAKE16 %s
define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; R600-LABEL: rotl_i32:
@@ -63,19 +65,42 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotl_i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_sub_i32 s3, 32, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s3, 31
-; GFX11-NEXT: s_mov_b32 s3, s2
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotl_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s3, 0, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s2, s2, v0.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotl_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s3, 0, s3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s2, s2, s3
+; GFX11-GISEL-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotl_i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_sub_i32 s3, 32, s3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_b32 s4, s3, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%0 = shl i32 %x, %y
%1 = sub i32 32, %y
@@ -165,26 +190,57 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotl_v2i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_sub_i32 s6, 32, s2
-; GFX11-NEXT: s_sub_i32 s7, 32, s3
-; GFX11-NEXT: s_mov_b32 s2, s1
-; GFX11-NEXT: s_mov_b32 s3, s1
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_and_b32 s6, s6, 31
-; GFX11-NEXT: s_and_b32 s7, s7, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotl_v2i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s2, 0, s2
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s3, 0, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s0, s0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v1, s1, s1, v1.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotl_v2i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s2, 0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s3, 0, s3
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s0, s0, s2
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s1, s1, s3
+; GFX11-GISEL-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotl_v2i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_sub_i32 s6, 32, s2
+; GFX11-SDAG-NEXT: s_sub_i32 s7, 32, s3
+; GFX11-SDAG-NEXT: s_mov_b32 s2, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX11-SDAG-NEXT: s_and_b32 s6, s6, 31
+; GFX11-SDAG-NEXT: s_and_b32 s7, s7, 31
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%0 = shl <2 x i32> %x, %y
%1 = sub <2 x i32> <i32 32, i32 32>, %y
@@ -313,36 +369,76 @@ define amdgpu_kernel void @rotl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotl_v4i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_sub_i32 s6, 32, s12
-; GFX11-NEXT: s_sub_i32 s7, 32, s13
-; GFX11-NEXT: s_sub_i32 s12, 32, s14
-; GFX11-NEXT: s_sub_i32 s13, 32, s15
-; GFX11-NEXT: s_mov_b32 s2, s11
-; GFX11-NEXT: s_mov_b32 s3, s11
-; GFX11-NEXT: s_mov_b32 s11, s10
-; GFX11-NEXT: s_mov_b32 s4, s9
-; GFX11-NEXT: s_mov_b32 s5, s9
-; GFX11-NEXT: s_mov_b32 s9, s8
-; GFX11-NEXT: s_and_b32 s13, s13, 31
-; GFX11-NEXT: s_and_b32 s12, s12, 31
-; GFX11-NEXT: s_and_b32 s14, s7, 31
-; GFX11-NEXT: s_and_b32 s15, s6, 31
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s13
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[10:11], s12
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s15
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotl_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s2, 0, s12
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s3, 0, s13
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s4, 0, s14
+; GFX11-GISEL-TRUE16-NEXT: s_sub_i32 s5, 0, s15
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s8, s8, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v1, s9, s9, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v2, s10, s10, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v3, s11, s11, v3.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotl_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s2, 0, s12
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s3, 0, s13
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s4, 0, s14
+; GFX11-GISEL-FAKE16-NEXT: s_sub_i32 s5, 0, s15
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s8, s8, s2
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s9, s9, s3
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v2, s10, s10, s4
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v3, s11, s11, s5
+; GFX11-GISEL-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotl_v4i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_sub_i32 s6, 32, s12
+; GFX11-SDAG-NEXT: s_sub_i32 s7, 32, s13
+; GFX11-SDAG-NEXT: s_sub_i32 s12, 32, s14
+; GFX11-SDAG-NEXT: s_sub_i32 s13, 32, s15
+; GFX11-SDAG-NEXT: s_mov_b32 s2, s11
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s11
+; GFX11-SDAG-NEXT: s_mov_b32 s11, s10
+; GFX11-SDAG-NEXT: s_mov_b32 s4, s9
+; GFX11-SDAG-NEXT: s_mov_b32 s5, s9
+; GFX11-SDAG-NEXT: s_mov_b32 s9, s8
+; GFX11-SDAG-NEXT: s_and_b32 s13, s13, 31
+; GFX11-SDAG-NEXT: s_and_b32 s12, s12, 31
+; GFX11-SDAG-NEXT: s_and_b32 s14, s7, 31
+; GFX11-SDAG-NEXT: s_and_b32 s15, s6, 31
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s13
+; GFX11-SDAG-NEXT: s_lshr_b64 s[6:7], s[10:11], s12
+; GFX11-SDAG-NEXT: s_lshr_b64 s[8:9], s[8:9], s15
+; GFX11-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%0 = shl <4 x i32> %x, %y
%1 = sub <4 x i32> <i32 32, i32 32, i32 32, i32 32>, %y
@@ -448,34 +544,69 @@ define void @test_rotl_i16(ptr addrspace(1) nocapture readonly %sourceA, ptr add
; GFX10-NEXT: global_store_short v[4:5], v0, off offset:8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_rotl_i16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_rotl_i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
+; GFX11-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 15
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b16 v1.l, v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_rotl_i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
+; GFX11-GISEL-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-GISEL-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b16 v2, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: test_rotl_i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
+; GFX11-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: test_rotl_i16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
-; GFX11-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, v2, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-FAKE16-LABEL: test_rotl_i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
+; GFX11-SDAG-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-SDAG-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_lshlrev_b16 v2, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%arrayidx = getelementptr inbounds i16, ptr addrspace(1) %sourceA, i64 16
%a = load i16, ptr addrspace(1) %arrayidx
@@ -486,3 +617,5 @@ entry:
store i16 %c, ptr addrspace(1) %arrayidx5
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-GISEL: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/rotr.ll b/llvm/test/CodeGen/AMDGPU/rotr.ll
index b25e58604b4e6..06ef4dc6ccc54 100644
--- a/llvm/test/CodeGen/AMDGPU/rotr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotr.ll
@@ -3,8 +3,10 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG,GFX11-SDAG-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG,GFX11-SDAG-FAKE16 %s
define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; R600-LABEL: rotr_i32:
@@ -58,17 +60,127 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotr_i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s4, s3, 31
-; GFX11-NEXT: s_mov_b32 s3, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotr_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s2, s2, v0.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotr_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s2, s2, s3
+; GFX11-GISEL-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotr_i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s4, s3, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
+entry:
+ %tmp0 = sub i32 32, %y
+ %tmp1 = shl i32 %x, %tmp0
+ %tmp2 = lshr i32 %x, %y
+ %tmp3 = or i32 %tmp1, %tmp2
+ store i32 %tmp3, ptr addrspace(1) %in
+ ret void
+}
+
+define amdgpu_kernel void @rotr_i32_sgpr(ptr addrspace(1) %in, i32 inreg %x, i32 inreg %y) {
+; R600-LABEL: rotr_i32_sgpr:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; R600-NEXT: BIT_ALIGN_INT * T1.X, KC0[2].Z, KC0[2].Z, KC0[2].W,
+;
+; SI-LABEL: rotr_i32_sgpr:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s3, 31
+; SI-NEXT: s_mov_b32 s3, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; GFX8-LABEL: rotr_i32_sgpr:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s4, s3, 31
+; GFX8-NEXT: s_mov_b32 s3, s2
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX10-LABEL: rotr_i32_sgpr:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_and_b32 s4, s3, 31
+; GFX10-NEXT: s_mov_b32 s3, s2
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: rotr_i32_sgpr:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s2, s2, v0.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotr_i32_sgpr:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s2, s2, s3
+; GFX11-GISEL-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotr_i32_sgpr:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s4, s3, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%tmp0 = sub i32 32, %y
%tmp1 = shl i32 %x, %tmp0
@@ -148,24 +260,50 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotr_v2i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s1
-; GFX11-NEXT: s_mov_b32 s7, s1
-; GFX11-NEXT: s_and_b32 s2, s2, 31
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_and_b32 s3, s3, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotr_v2i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s0, s0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v1, s1, s1, v1.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotr_v2i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s0, s0, s2
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s1, s1, s3
+; GFX11-GISEL-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotr_v2i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_mov_b32 s6, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s7, s1
+; GFX11-SDAG-NEXT: s_and_b32 s2, s2, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX11-SDAG-NEXT: s_and_b32 s3, s3, 31
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[6:7], s3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%tmp0 = sub <2 x i32> <i32 32, i32 32>, %y
%tmp1 = shl <2 x i32> %x, %tmp0
@@ -274,32 +412,64 @@ define amdgpu_kernel void @rotr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotr_v4i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s6, s15, 31
-; GFX11-NEXT: s_mov_b32 s2, s11
-; GFX11-NEXT: s_mov_b32 s3, s11
-; GFX11-NEXT: s_and_b32 s7, s14, 31
-; GFX11-NEXT: s_mov_b32 s11, s10
-; GFX11-NEXT: s_and_b32 s13, s13, 31
-; GFX11-NEXT: s_mov_b32 s4, s9
-; GFX11-NEXT: s_mov_b32 s5, s9
-; GFX11-NEXT: s_and_b32 s12, s12, 31
-; GFX11-NEXT: s_mov_b32 s9, s8
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[10:11], s7
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s12
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s13
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotr_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s8, s8, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v1, s9, s9, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v2, s10, s10, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v3, s11, s11, v3.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotr_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s8, s8, s12
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s9, s9, s13
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v2, s10, s10, s14
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v3, s11, s11, s15
+; GFX11-GISEL-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotr_v4i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s6, s15, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s2, s11
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s11
+; GFX11-SDAG-NEXT: s_and_b32 s7, s14, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s11, s10
+; GFX11-SDAG-NEXT: s_and_b32 s13, s13, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s4, s9
+; GFX11-SDAG-NEXT: s_mov_b32 s5, s9
+; GFX11-SDAG-NEXT: s_and_b32 s12, s12, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s9, s8
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX11-SDAG-NEXT: s_lshr_b64 s[6:7], s[10:11], s7
+; GFX11-SDAG-NEXT: s_lshr_b64 s[8:9], s[8:9], s12
+; GFX11-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s13
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%tmp0 = sub <4 x i32> <i32 32, i32 32, i32 32, i32 32>, %y
%tmp1 = shl <4 x i32> %x, %tmp0
@@ -476,49 +646,94 @@ define amdgpu_kernel void @rotr_v8i32(ptr addrspace(1) %in, <8 x i32> %x, <8 x i
; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: rotr_v8i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s19, s19, 31
-; GFX11-NEXT: s_mov_b32 s2, s11
-; GFX11-NEXT: s_mov_b32 s3, s11
-; GFX11-NEXT: s_and_b32 s17, s17, 31
-; GFX11-NEXT: s_mov_b32 s4, s9
-; GFX11-NEXT: s_mov_b32 s5, s9
-; GFX11-NEXT: s_and_b32 s16, s16, 31
-; GFX11-NEXT: s_mov_b32 s9, s8
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s19
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s17
-; GFX11-NEXT: s_and_b32 s23, s23, 31
-; GFX11-NEXT: s_mov_b32 s6, s15
-; GFX11-NEXT: s_mov_b32 s7, s15
-; GFX11-NEXT: s_and_b32 s22, s22, 31
-; GFX11-NEXT: s_mov_b32 s15, s14
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
-; GFX11-NEXT: s_mov_b32 s16, s13
-; GFX11-NEXT: s_mov_b32 s17, s13
-; GFX11-NEXT: s_and_b32 s3, s20, 31
-; GFX11-NEXT: s_mov_b32 s13, s12
-; GFX11-NEXT: s_and_b32 s5, s21, 31
-; GFX11-NEXT: s_and_b32 s18, s18, 31
-; GFX11-NEXT: s_mov_b32 s11, s10
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s23
-; GFX11-NEXT: s_lshr_b64 s[14:15], s[14:15], s22
-; GFX11-NEXT: s_lshr_b64 s[12:13], s[12:13], s3
-; GFX11-NEXT: s_lshr_b64 s[16:17], s[16:17], s5
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[10:11], s18
-; GFX11-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s16
-; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s6
-; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s4
-; GFX11-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s2
-; GFX11-NEXT: v_mov_b32_e32 v6, s10
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
-; GFX11-NEXT: global_store_b128 v8, v[4:7], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: rotr_v8i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s17
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s19
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v5, s21
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v7, s23
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v6, s22
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v0, s8, s8, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v1, s9, s9, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v2, s10, s10, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v3, s11, s11, v3.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v4, s12, s12, v4.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v5, s13, s13, v5.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v6, s14, s14, v6.l
+; GFX11-GISEL-TRUE16-NEXT: v_alignbit_b32 v7, s15, s15, v7.l
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX11-GISEL-TRUE16-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: rotr_v8i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v0, s8, s8, s16
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s9, s9, s17
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v2, s10, s10, s18
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v3, s11, s11, s19
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v4, s12, s12, s20
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v5, s13, s13, s21
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v6, s14, s14, s22
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v7, s15, s15, s23
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX11-GISEL-FAKE16-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: rotr_v8i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s19, s19, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s2, s11
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s11
+; GFX11-SDAG-NEXT: s_and_b32 s17, s17, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s4, s9
+; GFX11-SDAG-NEXT: s_mov_b32 s5, s9
+; GFX11-SDAG-NEXT: s_and_b32 s16, s16, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s9, s8
+; GFX11-SDAG-NEXT: s_lshr_b64 s[2:3], s[2:3], s19
+; GFX11-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s17
+; GFX11-SDAG-NEXT: s_and_b32 s23, s23, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s6, s15
+; GFX11-SDAG-NEXT: s_mov_b32 s7, s15
+; GFX11-SDAG-NEXT: s_and_b32 s22, s22, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s15, s14
+; GFX11-SDAG-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
+; GFX11-SDAG-NEXT: s_mov_b32 s16, s13
+; GFX11-SDAG-NEXT: s_mov_b32 s17, s13
+; GFX11-SDAG-NEXT: s_and_b32 s3, s20, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s13, s12
+; GFX11-SDAG-NEXT: s_and_b32 s5, s21, 31
+; GFX11-SDAG-NEXT: s_and_b32 s18, s18, 31
+; GFX11-SDAG-NEXT: s_mov_b32 s11, s10
+; GFX11-SDAG-NEXT: s_lshr_b64 s[6:7], s[6:7], s23
+; GFX11-SDAG-NEXT: s_lshr_b64 s[14:15], s[14:15], s22
+; GFX11-SDAG-NEXT: s_lshr_b64 s[12:13], s[12:13], s3
+; GFX11-SDAG-NEXT: s_lshr_b64 s[16:17], s[16:17], s5
+; GFX11-SDAG-NEXT: s_lshr_b64 s[10:11], s[10:11], s18
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s16
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s6
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v6, s10
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX11-SDAG-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
entry:
%tmp0 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %y
%tmp1 = shl <8 x i32> %x, %tmp0
@@ -621,34 +836,69 @@ define void @test_rotr_i16(ptr addrspace(1) nocapture readonly %sourceA, ptr add
; GFX10-NEXT: global_store_short v[4:5], v0, off offset:8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_rotr_i16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_rotr_i16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
-; GFX11-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, v2, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_rotr_i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
+; GFX11-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 15
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b16 v1.l, v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_rotr_i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
+; GFX11-GISEL-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-GISEL-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b16 v2, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: test_rotr_i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: global_load_d16_b16 v2, v[2:3], off offset:48
+; GFX11-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off offset:32
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_nc_u16 v0.h, 0, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-FAKE16-LABEL: test_rotr_i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: global_load_u16 v2, v[2:3], off offset:48
+; GFX11-SDAG-FAKE16-NEXT: global_load_u16 v0, v[0:1], off offset:32
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-SDAG-FAKE16-NEXT: v_sub_nc_u16 v1, 0, v2
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b16 v2, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%arrayidx = getelementptr inbounds i16, ptr addrspace(1) %sourceA, i64 16
%a = load i16, ptr addrspace(1) %arrayidx
@@ -659,3 +909,5 @@ entry:
store i16 %c, ptr addrspace(1) %arrayidx5
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-GISEL: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index fec3c1b3a6e1d..6d9f8b6b3c2c5 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -424,18 +424,18 @@ body: |
%264:vgpr_32 = V_LSHL_OR_B32_e64 %254, 8, %263, implicit $exec
%265:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%266:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %267:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %265.sub0, 0, 0, implicit $mode, implicit $exec
- %268:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %265.sub1, 0, 0, implicit $mode, implicit $exec
- %269:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %265.sub2, 0, 0, implicit $mode, implicit $exec
- %270:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %265.sub3, 0, 0, implicit $mode, implicit $exec
- %271:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %266.sub0, 0, 0, implicit $mode, implicit $exec
- %272:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %266.sub1, 0, 0, implicit $mode, implicit $exec
- %273:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %266.sub2, 0, 0, implicit $mode, implicit $exec
- %274:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %266.sub3, 0, 0, implicit $mode, implicit $exec
- undef %275.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %273, 0, %274, 0, 0, implicit $mode, implicit $exec
- %275.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %271, 0, %272, 0, 0, implicit $mode, implicit $exec
- %275.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %269, 0, %270, 0, 0, implicit $mode, implicit $exec
- %275.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %267, 0, %268, 0, 0, implicit $mode, implicit $exec
+ %267:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %265.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %268:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %265.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %269:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %265.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %270:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %265.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %271:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %266.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %272:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %266.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %273:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %266.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %274:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %266.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %275.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %273, 0, %274, 0, 0, implicit $mode, implicit $exec
+ %275.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %271, 0, %272, 0, 0, implicit $mode, implicit $exec
+ %275.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %269, 0, %270, 0, 0, implicit $mode, implicit $exec
+ %275.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %267, 0, %268, 0, 0, implicit $mode, implicit $exec
%4.sub2:sgpr_128 = S_LSHL_B32 %10, 1, implicit-def dead $scc
%4.sub3:sgpr_128 = COPY %171.sub3
%276:vgpr_32 = V_LSHLREV_B32_e64 1, %260.sub0, implicit $exec
@@ -446,18 +446,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %212.sub6, %212.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%277:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%278:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %279:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %277.sub0, 0, 0, implicit $mode, implicit $exec
- %280:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %277.sub1, 0, 0, implicit $mode, implicit $exec
- %281:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %277.sub2, 0, 0, implicit $mode, implicit $exec
- %282:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %277.sub3, 0, 0, implicit $mode, implicit $exec
- %283:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %278.sub0, 0, 0, implicit $mode, implicit $exec
- %284:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %278.sub1, 0, 0, implicit $mode, implicit $exec
- %285:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %278.sub2, 0, 0, implicit $mode, implicit $exec
- %286:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %278.sub3, 0, 0, implicit $mode, implicit $exec
- undef %287.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %285, 0, %286, 0, 0, implicit $mode, implicit $exec
- %287.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %283, 0, %284, 0, 0, implicit $mode, implicit $exec
- %287.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %281, 0, %282, 0, 0, implicit $mode, implicit $exec
- %287.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %279, 0, %280, 0, 0, implicit $mode, implicit $exec
+ %279:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %277.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %280:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %277.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %281:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %277.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %282:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %277.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %283:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %278.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %284:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %278.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %285:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %278.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %286:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %278.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %287.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %285, 0, %286, 0, 0, implicit $mode, implicit $exec
+ %287.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %283, 0, %284, 0, 0, implicit $mode, implicit $exec
+ %287.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %281, 0, %282, 0, 0, implicit $mode, implicit $exec
+ %287.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %279, 0, %280, 0, 0, implicit $mode, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %287, %276, %4, 0, 128, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %246.sub0, %246.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
DS_WRITE2ST64_B32_gfx9 %262, %246.sub2, %246.sub3, 2, 3, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -465,18 +465,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %246.sub6, %246.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%288:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%289:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %290:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %288.sub0, 0, 0, implicit $mode, implicit $exec
- %291:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %288.sub1, 0, 0, implicit $mode, implicit $exec
- %292:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %288.sub2, 0, 0, implicit $mode, implicit $exec
- %293:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %288.sub3, 0, 0, implicit $mode, implicit $exec
- %294:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %289.sub0, 0, 0, implicit $mode, implicit $exec
- %295:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %289.sub1, 0, 0, implicit $mode, implicit $exec
- %296:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %289.sub2, 0, 0, implicit $mode, implicit $exec
- %297:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %289.sub3, 0, 0, implicit $mode, implicit $exec
- undef %298.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %296, 0, %297, 0, 0, implicit $mode, implicit $exec
- %298.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %294, 0, %295, 0, 0, implicit $mode, implicit $exec
- %298.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %292, 0, %293, 0, 0, implicit $mode, implicit $exec
- %298.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %290, 0, %291, 0, 0, implicit $mode, implicit $exec
+ %290:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %288.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %291:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %288.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %292:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %288.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %293:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %288.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %294:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %289.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %295:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %289.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %296:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %289.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %297:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %289.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %298.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %296, 0, %297, 0, 0, implicit $mode, implicit $exec
+ %298.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %294, 0, %295, 0, 0, implicit $mode, implicit $exec
+ %298.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %292, 0, %293, 0, 0, implicit $mode, implicit $exec
+ %298.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %290, 0, %291, 0, 0, implicit $mode, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %298, %276, %4, 0, 256, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
%299:vgpr_32 = V_ADD_U32_e64 192, %260.sub0, 0, implicit $exec
DS_WRITE2ST64_B32_gfx9 %262, %250.sub0, %250.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -485,18 +485,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %250.sub6, %250.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%300:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%301:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %302:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %300.sub0, 0, 0, implicit $mode, implicit $exec
- %303:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %300.sub1, 0, 0, implicit $mode, implicit $exec
- %304:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %300.sub2, 0, 0, implicit $mode, implicit $exec
- %305:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %300.sub3, 0, 0, implicit $mode, implicit $exec
- %306:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %301.sub0, 0, 0, implicit $mode, implicit $exec
- %307:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %301.sub1, 0, 0, implicit $mode, implicit $exec
- %308:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %301.sub2, 0, 0, implicit $mode, implicit $exec
- %309:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %301.sub3, 0, 0, implicit $mode, implicit $exec
- undef %310.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %308, 0, %309, 0, 0, implicit $mode, implicit $exec
- %310.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %306, 0, %307, 0, 0, implicit $mode, implicit $exec
- %310.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %304, 0, %305, 0, 0, implicit $mode, implicit $exec
- %310.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %302, 0, %303, 0, 0, implicit $mode, implicit $exec
+ %302:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %300.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %303:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %300.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %304:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %300.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %305:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %300.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %306:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %301.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %307:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %301.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %308:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %301.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %309:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %301.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %310.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %308, 0, %309, 0, 0, implicit $mode, implicit $exec
+ %310.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %306, 0, %307, 0, 0, implicit $mode, implicit $exec
+ %310.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %304, 0, %305, 0, 0, implicit $mode, implicit $exec
+ %310.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %302, 0, %303, 0, 0, implicit $mode, implicit $exec
%311:vgpr_32 = V_LSHLREV_B32_e64 1, %299, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %310, %311, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
%312:sreg_32 = nsw S_LSHL_B32 %9, 5, implicit-def dead $scc
@@ -506,18 +506,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %253.sub6, %253.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%313:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%314:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %315:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %313.sub0, 0, 0, implicit $mode, implicit $exec
- %316:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %313.sub1, 0, 0, implicit $mode, implicit $exec
- %317:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %313.sub2, 0, 0, implicit $mode, implicit $exec
- %318:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %313.sub3, 0, 0, implicit $mode, implicit $exec
- %319:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %314.sub0, 0, 0, implicit $mode, implicit $exec
- %320:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %314.sub1, 0, 0, implicit $mode, implicit $exec
- %321:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %314.sub2, 0, 0, implicit $mode, implicit $exec
- %322:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %314.sub3, 0, 0, implicit $mode, implicit $exec
- undef %323.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %321, 0, %322, 0, 0, implicit $mode, implicit $exec
- %323.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %319, 0, %320, 0, 0, implicit $mode, implicit $exec
- %323.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %317, 0, %318, 0, 0, implicit $mode, implicit $exec
- %323.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %315, 0, %316, 0, 0, implicit $mode, implicit $exec
+ %315:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %313.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %316:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %313.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %317:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %313.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %318:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %313.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %319:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %314.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %320:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %314.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %321:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %314.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %322:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %314.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %323.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %321, 0, %322, 0, 0, implicit $mode, implicit $exec
+ %323.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %319, 0, %320, 0, 0, implicit $mode, implicit $exec
+ %323.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %317, 0, %318, 0, 0, implicit $mode, implicit $exec
+ %323.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %315, 0, %316, 0, 0, implicit $mode, implicit $exec
%324:vgpr_32 = V_ADD_LSHL_U32_e64 %299, %312, 1, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %323, %324, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %214.sub0, %214.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -526,18 +526,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %214.sub6, %214.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%325:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%326:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %327:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %325.sub0, 0, 0, implicit $mode, implicit $exec
- %328:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %325.sub1, 0, 0, implicit $mode, implicit $exec
- %329:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %325.sub2, 0, 0, implicit $mode, implicit $exec
- %330:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %325.sub3, 0, 0, implicit $mode, implicit $exec
- %331:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %326.sub0, 0, 0, implicit $mode, implicit $exec
- %332:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %326.sub1, 0, 0, implicit $mode, implicit $exec
- %333:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %326.sub2, 0, 0, implicit $mode, implicit $exec
- %334:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %326.sub3, 0, 0, implicit $mode, implicit $exec
- undef %335.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %333, 0, %334, 0, 0, implicit $mode, implicit $exec
- %335.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %331, 0, %332, 0, 0, implicit $mode, implicit $exec
- %335.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %329, 0, %330, 0, 0, implicit $mode, implicit $exec
- %335.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %327, 0, %328, 0, 0, implicit $mode, implicit $exec
+ %327:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %325.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %328:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %325.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %329:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %325.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %330:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %325.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %331:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %326.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %332:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %326.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %333:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %326.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %334:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %326.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %335.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %333, 0, %334, 0, 0, implicit $mode, implicit $exec
+ %335.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %331, 0, %332, 0, 0, implicit $mode, implicit $exec
+ %335.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %329, 0, %330, 0, 0, implicit $mode, implicit $exec
+ %335.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %327, 0, %328, 0, 0, implicit $mode, implicit $exec
%336:vgpr_32 = V_ADD_U32_e64 -128, %324, 0, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %335, %336, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %247.sub0, %247.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -546,18 +546,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %247.sub6, %247.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%337:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%338:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %339:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %337.sub0, 0, 0, implicit $mode, implicit $exec
- %340:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %337.sub1, 0, 0, implicit $mode, implicit $exec
- %341:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %337.sub2, 0, 0, implicit $mode, implicit $exec
- %342:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %337.sub3, 0, 0, implicit $mode, implicit $exec
- %343:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %338.sub0, 0, 0, implicit $mode, implicit $exec
- %344:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %338.sub1, 0, 0, implicit $mode, implicit $exec
- %345:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %338.sub2, 0, 0, implicit $mode, implicit $exec
- %346:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %338.sub3, 0, 0, implicit $mode, implicit $exec
- undef %347.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %345, 0, %346, 0, 0, implicit $mode, implicit $exec
- %347.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %343, 0, %344, 0, 0, implicit $mode, implicit $exec
- %347.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %341, 0, %342, 0, 0, implicit $mode, implicit $exec
- %347.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %339, 0, %340, 0, 0, implicit $mode, implicit $exec
+ %339:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %337.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %340:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %337.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %341:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %337.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %342:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %337.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %343:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %338.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %344:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %338.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %345:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %338.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %346:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %338.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %347.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %345, 0, %346, 0, 0, implicit $mode, implicit $exec
+ %347.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %343, 0, %344, 0, 0, implicit $mode, implicit $exec
+ %347.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %341, 0, %342, 0, 0, implicit $mode, implicit $exec
+ %347.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %339, 0, %340, 0, 0, implicit $mode, implicit $exec
%348:vgpr_32 = V_ADD_U32_e64 -256, %324, 0, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %347, %348, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
%349:vgpr_32 = V_ADD_U32_e64 %312, %260.sub0, 0, implicit $exec
@@ -567,18 +567,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %213.sub6, %213.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%350:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%351:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %352:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %350.sub0, 0, 0, implicit $mode, implicit $exec
- %353:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %350.sub1, 0, 0, implicit $mode, implicit $exec
- %354:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %350.sub2, 0, 0, implicit $mode, implicit $exec
- %355:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %350.sub3, 0, 0, implicit $mode, implicit $exec
- %356:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %351.sub0, 0, 0, implicit $mode, implicit $exec
- %357:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %351.sub1, 0, 0, implicit $mode, implicit $exec
- %358:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %351.sub2, 0, 0, implicit $mode, implicit $exec
- %359:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %351.sub3, 0, 0, implicit $mode, implicit $exec
- undef %360.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %358, 0, %359, 0, 0, implicit $mode, implicit $exec
- %360.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %356, 0, %357, 0, 0, implicit $mode, implicit $exec
- %360.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %354, 0, %355, 0, 0, implicit $mode, implicit $exec
- %360.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %352, 0, %353, 0, 0, implicit $mode, implicit $exec
+ %352:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %350.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %353:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %350.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %354:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %350.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %355:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %350.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %356:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %351.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %357:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %351.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %358:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %351.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %359:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %351.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %360.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %358, 0, %359, 0, 0, implicit $mode, implicit $exec
+ %360.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %356, 0, %357, 0, 0, implicit $mode, implicit $exec
+ %360.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %354, 0, %355, 0, 0, implicit $mode, implicit $exec
+ %360.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %352, 0, %353, 0, 0, implicit $mode, implicit $exec
%361:vgpr_32 = V_LSHLREV_B32_e64 1, %349, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %360, %361, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
%362:vgpr_32 = V_ADD_U32_e64 %312, %349, 0, implicit $exec
@@ -588,18 +588,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %216.sub6, %216.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%363:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%364:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %365:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %363.sub0, 0, 0, implicit $mode, implicit $exec
- %366:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %363.sub1, 0, 0, implicit $mode, implicit $exec
- %367:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %363.sub2, 0, 0, implicit $mode, implicit $exec
- %368:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %363.sub3, 0, 0, implicit $mode, implicit $exec
- %369:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %364.sub0, 0, 0, implicit $mode, implicit $exec
- %370:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %364.sub1, 0, 0, implicit $mode, implicit $exec
- %371:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %364.sub2, 0, 0, implicit $mode, implicit $exec
- %372:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %364.sub3, 0, 0, implicit $mode, implicit $exec
- undef %373.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %371, 0, %372, 0, 0, implicit $mode, implicit $exec
- %373.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %369, 0, %370, 0, 0, implicit $mode, implicit $exec
- %373.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %367, 0, %368, 0, 0, implicit $mode, implicit $exec
- %373.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %365, 0, %366, 0, 0, implicit $mode, implicit $exec
+ %365:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %363.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %366:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %363.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %367:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %363.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %368:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %363.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %369:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %364.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %370:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %364.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %371:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %364.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %372:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %364.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %373.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %371, 0, %372, 0, 0, implicit $mode, implicit $exec
+ %373.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %369, 0, %370, 0, 0, implicit $mode, implicit $exec
+ %373.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %367, 0, %368, 0, 0, implicit $mode, implicit $exec
+ %373.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %365, 0, %366, 0, 0, implicit $mode, implicit $exec
%374:vgpr_32 = V_LSHLREV_B32_e64 1, %362, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %373, %374, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %248.sub0, %248.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -608,18 +608,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %248.sub6, %248.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%375:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%376:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %377:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %375.sub0, 0, 0, implicit $mode, implicit $exec
- %378:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %375.sub1, 0, 0, implicit $mode, implicit $exec
- %379:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %375.sub2, 0, 0, implicit $mode, implicit $exec
- %380:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %375.sub3, 0, 0, implicit $mode, implicit $exec
- %381:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %376.sub0, 0, 0, implicit $mode, implicit $exec
- %382:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %376.sub1, 0, 0, implicit $mode, implicit $exec
- %383:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %376.sub2, 0, 0, implicit $mode, implicit $exec
- %384:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %376.sub3, 0, 0, implicit $mode, implicit $exec
- undef %385.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %383, 0, %384, 0, 0, implicit $mode, implicit $exec
- %385.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %381, 0, %382, 0, 0, implicit $mode, implicit $exec
- %385.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %379, 0, %380, 0, 0, implicit $mode, implicit $exec
- %385.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %377, 0, %378, 0, 0, implicit $mode, implicit $exec
+ %377:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %375.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %378:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %375.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %379:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %375.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %380:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %375.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %381:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %376.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %382:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %376.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %383:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %376.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %384:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %376.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %385.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %383, 0, %384, 0, 0, implicit $mode, implicit $exec
+ %385.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %381, 0, %382, 0, 0, implicit $mode, implicit $exec
+ %385.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %379, 0, %380, 0, 0, implicit $mode, implicit $exec
+ %385.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %377, 0, %378, 0, 0, implicit $mode, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %385, %374, %4, 0, 128, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %217.sub0, %217.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
DS_WRITE2ST64_B32_gfx9 %262, %217.sub2, %217.sub3, 2, 3, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -627,18 +627,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %217.sub6, %217.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%386:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%387:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %388:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %386.sub0, 0, 0, implicit $mode, implicit $exec
- %389:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %386.sub1, 0, 0, implicit $mode, implicit $exec
- %390:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %386.sub2, 0, 0, implicit $mode, implicit $exec
- %391:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %386.sub3, 0, 0, implicit $mode, implicit $exec
- %392:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %387.sub0, 0, 0, implicit $mode, implicit $exec
- %393:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %387.sub1, 0, 0, implicit $mode, implicit $exec
- %394:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %387.sub2, 0, 0, implicit $mode, implicit $exec
- %395:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %387.sub3, 0, 0, implicit $mode, implicit $exec
- undef %396.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %394, 0, %395, 0, 0, implicit $mode, implicit $exec
- %396.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %392, 0, %393, 0, 0, implicit $mode, implicit $exec
- %396.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %390, 0, %391, 0, 0, implicit $mode, implicit $exec
- %396.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %388, 0, %389, 0, 0, implicit $mode, implicit $exec
+ %388:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %386.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %389:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %386.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %390:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %386.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %391:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %386.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %392:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %387.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %393:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %387.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %394:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %387.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %395:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %387.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %396.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %394, 0, %395, 0, 0, implicit $mode, implicit $exec
+ %396.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %392, 0, %393, 0, 0, implicit $mode, implicit $exec
+ %396.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %390, 0, %391, 0, 0, implicit $mode, implicit $exec
+ %396.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %388, 0, %389, 0, 0, implicit $mode, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %396, %374, %4, 0, 256, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
%397:vgpr_32 = V_ADD_U32_e64 192, %362, 0, implicit $exec
DS_WRITE2ST64_B32_gfx9 %262, %251.sub0, %251.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -647,18 +647,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %251.sub6, %251.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%398:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%399:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %400:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %398.sub0, 0, 0, implicit $mode, implicit $exec
- %401:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %398.sub1, 0, 0, implicit $mode, implicit $exec
- %402:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %398.sub2, 0, 0, implicit $mode, implicit $exec
- %403:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %398.sub3, 0, 0, implicit $mode, implicit $exec
- %404:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %399.sub0, 0, 0, implicit $mode, implicit $exec
- %405:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %399.sub1, 0, 0, implicit $mode, implicit $exec
- %406:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %399.sub2, 0, 0, implicit $mode, implicit $exec
- %407:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %399.sub3, 0, 0, implicit $mode, implicit $exec
- undef %408.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %406, 0, %407, 0, 0, implicit $mode, implicit $exec
- %408.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %404, 0, %405, 0, 0, implicit $mode, implicit $exec
- %408.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %402, 0, %403, 0, 0, implicit $mode, implicit $exec
- %408.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %400, 0, %401, 0, 0, implicit $mode, implicit $exec
+ %400:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %398.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %401:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %398.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %402:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %398.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %403:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %398.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %404:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %399.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %405:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %399.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %406:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %399.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %407:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %399.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %408.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %406, 0, %407, 0, 0, implicit $mode, implicit $exec
+ %408.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %404, 0, %405, 0, 0, implicit $mode, implicit $exec
+ %408.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %402, 0, %403, 0, 0, implicit $mode, implicit $exec
+ %408.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %400, 0, %401, 0, 0, implicit $mode, implicit $exec
%409:vgpr_32 = V_LSHLREV_B32_e64 1, %397, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %408, %409, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %252.sub0, %252.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -667,18 +667,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %252.sub6, %252.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%410:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%411:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %412:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %410.sub0, 0, 0, implicit $mode, implicit $exec
- %413:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %410.sub1, 0, 0, implicit $mode, implicit $exec
- %414:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %410.sub2, 0, 0, implicit $mode, implicit $exec
- %415:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %410.sub3, 0, 0, implicit $mode, implicit $exec
- %416:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %411.sub0, 0, 0, implicit $mode, implicit $exec
- %417:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %411.sub1, 0, 0, implicit $mode, implicit $exec
- %418:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %411.sub2, 0, 0, implicit $mode, implicit $exec
- %419:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %411.sub3, 0, 0, implicit $mode, implicit $exec
- undef %420.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %418, 0, %419, 0, 0, implicit $mode, implicit $exec
- %420.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %416, 0, %417, 0, 0, implicit $mode, implicit $exec
- %420.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %414, 0, %415, 0, 0, implicit $mode, implicit $exec
- %420.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %412, 0, %413, 0, 0, implicit $mode, implicit $exec
+ %412:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %410.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %413:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %410.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %414:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %410.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %415:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %410.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %416:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %411.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %417:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %411.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %418:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %411.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %419:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %411.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %420.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %418, 0, %419, 0, 0, implicit $mode, implicit $exec
+ %420.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %416, 0, %417, 0, 0, implicit $mode, implicit $exec
+ %420.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %414, 0, %415, 0, 0, implicit $mode, implicit $exec
+ %420.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %412, 0, %413, 0, 0, implicit $mode, implicit $exec
%421:vgpr_32 = V_ADD_LSHL_U32_e64 %397, %312, 1, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %420, %421, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %220.sub0, %220.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -687,18 +687,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %220.sub6, %220.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%422:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%423:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %424:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %422.sub0, 0, 0, implicit $mode, implicit $exec
- %425:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %422.sub1, 0, 0, implicit $mode, implicit $exec
- %426:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %422.sub2, 0, 0, implicit $mode, implicit $exec
- %427:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %422.sub3, 0, 0, implicit $mode, implicit $exec
- %428:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %423.sub0, 0, 0, implicit $mode, implicit $exec
- %429:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %423.sub1, 0, 0, implicit $mode, implicit $exec
- %430:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %423.sub2, 0, 0, implicit $mode, implicit $exec
- %431:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %423.sub3, 0, 0, implicit $mode, implicit $exec
- undef %432.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %430, 0, %431, 0, 0, implicit $mode, implicit $exec
- %432.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %428, 0, %429, 0, 0, implicit $mode, implicit $exec
- %432.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %426, 0, %427, 0, 0, implicit $mode, implicit $exec
- %432.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %424, 0, %425, 0, 0, implicit $mode, implicit $exec
+ %424:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %422.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %425:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %422.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %426:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %422.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %427:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %422.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %428:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %423.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %429:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %423.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %430:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %423.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %431:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %423.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %432.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %430, 0, %431, 0, 0, implicit $mode, implicit $exec
+ %432.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %428, 0, %429, 0, 0, implicit $mode, implicit $exec
+ %432.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %426, 0, %427, 0, 0, implicit $mode, implicit $exec
+ %432.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %424, 0, %425, 0, 0, implicit $mode, implicit $exec
%433:vgpr_32 = V_ADD_U32_e64 -128, %421, 0, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %432, %433, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %249.sub0, %249.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -707,18 +707,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %249.sub6, %249.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%434:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%435:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %436:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %434.sub0, 0, 0, implicit $mode, implicit $exec
- %437:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %434.sub1, 0, 0, implicit $mode, implicit $exec
- %438:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %434.sub2, 0, 0, implicit $mode, implicit $exec
- %439:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %434.sub3, 0, 0, implicit $mode, implicit $exec
- %440:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %435.sub0, 0, 0, implicit $mode, implicit $exec
- %441:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %435.sub1, 0, 0, implicit $mode, implicit $exec
- %442:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %435.sub2, 0, 0, implicit $mode, implicit $exec
- %443:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %435.sub3, 0, 0, implicit $mode, implicit $exec
- undef %444.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %442, 0, %443, 0, 0, implicit $mode, implicit $exec
- %444.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %440, 0, %441, 0, 0, implicit $mode, implicit $exec
- %444.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %438, 0, %439, 0, 0, implicit $mode, implicit $exec
- %444.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %436, 0, %437, 0, 0, implicit $mode, implicit $exec
+ %436:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %434.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %437:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %434.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %438:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %434.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %439:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %434.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %440:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %435.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %441:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %435.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %442:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %435.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %443:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %435.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %444.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %442, 0, %443, 0, 0, implicit $mode, implicit $exec
+ %444.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %440, 0, %441, 0, 0, implicit $mode, implicit $exec
+ %444.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %438, 0, %439, 0, 0, implicit $mode, implicit $exec
+ %444.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %436, 0, %437, 0, 0, implicit $mode, implicit $exec
%445:vgpr_32 = V_ADD_U32_e64 -256, %421, 0, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %444, %445, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
DS_WRITE2ST64_B32_gfx9 %262, %219.sub0, %219.sub1, 0, 1, 0, implicit $exec :: (store (s32), addrspace 3)
@@ -727,18 +727,18 @@ body: |
DS_WRITE2ST64_B32_gfx9 %262, %219.sub6, %219.sub7, 6, 7, 0, implicit $exec :: (store (s32), addrspace 3)
%446:vreg_128 = DS_READ_B128_gfx9 %264, 0, 0, implicit $exec :: (load (s128), addrspace 3)
%447:vreg_128 = DS_READ_B128_gfx9 %264, 16, 0, implicit $exec :: (load (s128), addrspace 3)
- %448:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %446.sub0, 0, 0, implicit $mode, implicit $exec
- %449:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %446.sub1, 0, 0, implicit $mode, implicit $exec
- %450:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %446.sub2, 0, 0, implicit $mode, implicit $exec
- %451:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %446.sub3, 0, 0, implicit $mode, implicit $exec
- %452:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %447.sub0, 0, 0, implicit $mode, implicit $exec
- %453:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %447.sub1, 0, 0, implicit $mode, implicit $exec
- %454:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %447.sub2, 0, 0, implicit $mode, implicit $exec
- %455:vgpr_32 = V_CVT_F16_F32_fake16_e64 0, %447.sub3, 0, 0, implicit $mode, implicit $exec
- undef %456.sub3:vreg_128 = V_PACK_B32_F16_e64 0, %454, 0, %455, 0, 0, implicit $mode, implicit $exec
- %456.sub2:vreg_128 = V_PACK_B32_F16_e64 0, %452, 0, %453, 0, 0, implicit $mode, implicit $exec
- %456.sub1:vreg_128 = V_PACK_B32_F16_e64 0, %450, 0, %451, 0, 0, implicit $mode, implicit $exec
- %456.sub0:vreg_128 = V_PACK_B32_F16_e64 0, %448, 0, %449, 0, 0, implicit $mode, implicit $exec
+ %448:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %446.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %449:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %446.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %450:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %446.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %451:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %446.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ %452:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %447.sub0, 0, 0, 0, implicit $mode, implicit $exec
+ %453:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %447.sub1, 0, 0, 0, implicit $mode, implicit $exec
+ %454:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %447.sub2, 0, 0, 0, implicit $mode, implicit $exec
+ %455:vgpr_16 = V_CVT_F16_F32_t16_e64 0, %447.sub3, 0, 0, 0, implicit $mode, implicit $exec
+ undef %456.sub3:vreg_128 = V_PACK_B32_F16_t16_e64 0, %454, 0, %455, 0, 0, implicit $mode, implicit $exec
+ %456.sub2:vreg_128 = V_PACK_B32_F16_t16_e64 0, %452, 0, %453, 0, 0, implicit $mode, implicit $exec
+ %456.sub1:vreg_128 = V_PACK_B32_F16_t16_e64 0, %450, 0, %451, 0, 0, implicit $mode, implicit $exec
+ %456.sub0:vreg_128 = V_PACK_B32_F16_t16_e64 0, %448, 0, %449, 0, 0, implicit $mode, implicit $exec
%457:vgpr_32 = V_ADD_LSHL_U32_e64 %362, %312, 1, implicit $exec
BUFFER_STORE_DWORDX4_OFFEN_exact %456, %457, %4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-mad-fma.mir b/llvm/test/CodeGen/AMDGPU/shrink-mad-fma.mir
index 910fa45b4b882..f10fc72d51ee8 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-mad-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-mad-fma.mir
@@ -1,10 +1,17 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgpu10.10 -run-pass si-shrink-instructions -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=GFX10
# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -run-pass si-shrink-instructions -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=GFX11
---
name: mad_cvv_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_cvv_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADMK_F32 $vgpr0, 1092616192, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_cvv_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -20,6 +27,12 @@ body: |
name: mad_vcv_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vcv_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADMK_F32 $vgpr0, 1092616192, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vcv_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -35,6 +48,12 @@ body: |
name: mad_vvc_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vvc_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADAK_F32 $vgpr0, $vgpr1, 1092616192, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vvc_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -50,6 +69,12 @@ body: |
name: mad_vsc_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vsc_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $sgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADAK_F32 $vgpr0, $vgpr1, 1092616192, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vsc_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $sgpr1 = IMPLICIT_DEF
@@ -65,6 +90,12 @@ body: |
name: fma_cvv_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_cvv_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_FMAMK_F32 $vgpr0, 1092616192, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_cvv_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -80,6 +111,12 @@ body: |
name: fma_vcv_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vcv_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_FMAMK_F32 $vgpr0, 1092616192, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vcv_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -95,6 +132,12 @@ body: |
name: fma_vvc_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vvc_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_FMAAK_F32 $vgpr0, $vgpr1, 1092616192, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vvc_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -110,6 +153,12 @@ body: |
name: fma_vsc_f32
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vsc_f32
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $sgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_FMAAK_F32 $vgpr0, $vgpr1, 1092616192, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vsc_f32
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $sgpr1 = IMPLICIT_DEF
@@ -125,6 +174,12 @@ body: |
name: mad_cvv_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_cvv_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADMK_F16 $vgpr0, 18688, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_cvv_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -140,6 +195,12 @@ body: |
name: mad_vcv_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vcv_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADMK_F16 $vgpr0, 18688, $vgpr1, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vcv_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -155,6 +216,12 @@ body: |
name: mad_vvc_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vvc_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADAK_F16 $vgpr0, $vgpr1, 18688, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vvc_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -170,6 +237,12 @@ body: |
name: mad_vsc_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: mad_vsc_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $sgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2 = V_MADAK_F16 $vgpr0, $vgpr1, 18688, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: mad_vsc_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $sgpr1 = IMPLICIT_DEF
@@ -185,6 +258,12 @@ body: |
name: fma_cvv_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_cvv_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2_lo16 = V_FMAMK_F16_t16 $vgpr0_lo16, 18688, $vgpr1_lo16, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_cvv_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -200,6 +279,12 @@ body: |
name: fma_vcv_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vcv_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2_lo16 = V_FMAMK_F16_t16 $vgpr0_lo16, 18688, $vgpr1_lo16, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vcv_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -215,6 +300,12 @@ body: |
name: fma_vvc_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vvc_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2_lo16 = V_FMAAK_F16_t16 $vgpr0_lo16, $vgpr1_lo16, 18688, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vvc_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $vgpr1 = IMPLICIT_DEF
@@ -230,6 +321,12 @@ body: |
name: fma_vsc_f16
body: |
bb.0:
+ ; GFX10-LABEL: name: fma_vsc_f16
+ ; GFX10: $vgpr0 = IMPLICIT_DEF
+ ; GFX10-NEXT: $sgpr1 = IMPLICIT_DEF
+ ; GFX10-NEXT: $vgpr2_lo16 = V_FMAAK_F16_t16 $vgpr0_hi16, $vgpr1_hi16, 18688, implicit $mode, implicit $exec
+ ; GFX10-NEXT: SI_RETURN implicit $vgpr2
+ ;
; GFX11-LABEL: name: fma_vsc_f16
; GFX11: $vgpr0 = IMPLICIT_DEF
; GFX11-NEXT: $sgpr1 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-true16.mir b/llvm/test/CodeGen/AMDGPU/shrink-true16.mir
index 9db7969976615..8fa1c2efb1fcc 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-true16.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-true16.mir
@@ -11,8 +11,8 @@ body: |
; GFX1100-LABEL: name: 16bit_lo128_shrink
; GFX1100: liveins: $vgpr127
; GFX1100-NEXT: {{ $}}
- ; GFX1100-NEXT: V_CMP_EQ_U16_fake16_e32 0, $vgpr127, implicit-def $vcc_lo, implicit $exec, implicit $exec
- $vcc_lo = V_CMP_EQ_U16_fake16_e64 0, $vgpr127, implicit-def $vcc, implicit $exec
+ ; GFX1100-NEXT: V_CMP_EQ_U16_t16_e32 $vgpr127_lo16, $vgpr127_hi16, implicit-def $vcc_lo, implicit $exec
+ $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr127_lo16, 0, $vgpr127_hi16, 0, implicit $exec
...
---
@@ -24,6 +24,32 @@ body: |
; GFX1100-LABEL: name: 16bit_lo128_no_shrink
; GFX1100: liveins: $vgpr128
; GFX1100-NEXT: {{ $}}
- ; GFX1100-NEXT: $vcc_lo = V_CMP_EQ_U16_fake16_e64 0, $vgpr128, implicit-def $vcc_lo, implicit $exec
- $vcc_lo = V_CMP_EQ_U16_fake16_e64 0, $vgpr128, implicit-def $vcc, implicit $exec
+ ; GFX1100-NEXT: $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr128_lo16, 0, $vgpr128_hi16, 0, implicit $exec
+ $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr128_lo16, 0, $vgpr128_hi16, 0, implicit $exec
+...
+
+---
+name: 32bit_lo128_shrink
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr127
+ ; GFX1100-LABEL: name: 32bit_lo128_shrink
+ ; GFX1100: liveins: $vgpr127
+ ; GFX1100-NEXT: {{ $}}
+ ; GFX1100-NEXT: V_CMP_EQ_U16_fake16_e32 $vgpr127, $vgpr127, implicit-def $vcc_lo, implicit $exec
+ $vcc_lo = V_CMP_EQ_U16_fake16_e64 $vgpr127, $vgpr127, implicit $exec
+...
+
+---
+name: 32bit_lo128_no_shrink
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr128
+ ; GFX1100-LABEL: name: 32bit_lo128_no_shrink
+ ; GFX1100: liveins: $vgpr128
+ ; GFX1100-NEXT: {{ $}}
+ ; GFX1100-NEXT: $vcc_lo = V_CMP_EQ_U16_fake16_e64 $vgpr128, $vgpr128, implicit $exec
+ $vcc_lo = V_CMP_EQ_U16_fake16_e64 $vgpr128, $vgpr128, implicit $exec
...
diff --git a/llvm/test/CodeGen/AMDGPU/shrink16.mir b/llvm/test/CodeGen/AMDGPU/shrink16.mir
new file mode 100644
index 0000000000000..a08c6eb1e290c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/shrink16.mir
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=+real-true16 %s -run-pass=si-shrink-instructions -verify-machineinstrs -o - | FileCheck -check-prefixes=GCN %s
+
+
+# Should not shrink 16 bit instructions preRA
+---
+name: preRA
+tracksRegLiveness: true
+body: |
+ bb.0:
+
+ ; GCN-LABEL: name: preRA
+ ; GCN: [[DEF:%[0-9]+]]:vreg_128 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[DEF]].sub1, implicit $exec
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub3
+ ; GCN-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_F16_t16_e64 0, [[V_MOV_B32_e32_]].hi16, 0, [[COPY]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0, implicit [[V_ADD_F16_t16_e64_]]
+ %0:vreg_128 = IMPLICIT_DEF
+ %3:vgpr_32 = V_MOV_B32_e32 %0.sub1, implicit $exec
+ %4:vgpr_32 = COPY %0.sub3
+ %5:vgpr_16 = V_ADD_F16_t16_e64 0, %3.hi16, 0, %4.lo16, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %5
+
+...
+
+---
+name: postRA
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: postRA
+ ; GCN: renamable $vgpr1 = V_MOV_B32_e32 undef $vgpr1, implicit $exec
+ ; GCN-NEXT: renamable $vgpr1_hi16 = V_ADD_F16_t16_e32 $vgpr1_hi16, undef $vgpr1_lo16, implicit $mode, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr1_hi16
+ renamable $vgpr1 = V_MOV_B32_e32 undef $vgpr1, implicit $exec
+ renamable $vgpr1_hi16 = V_ADD_F16_t16_e64 0, $vgpr1_hi16, 0, undef $vgpr1_lo16, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit killed renamable $vgpr1_hi16
+...
+
+# Should not shrink 16 bit instructions if they use vgpr not addressable in 32bit encoding
+---
+name: postRA_noShrink
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: postRA_noShrink
+ ; GCN: renamable $vgpr1 = V_MOV_B32_e32 undef $vgpr1, implicit $exec
+ ; GCN-NEXT: renamable $vgpr1_hi16 = V_ADD_F16_t16_e64 0, $vgpr199_hi16, 0, undef $vgpr1_lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr1_hi16
+ renamable $vgpr1 = V_MOV_B32_e32 undef $vgpr1, implicit $exec
+ renamable $vgpr1_hi16 = V_ADD_F16_t16_e64 0, $vgpr199_hi16, 0, undef $vgpr1_lo16, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit killed renamable $vgpr1_hi16
+...
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 32f014bfdda57..b9fa6a346cd61 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -249,20 +249,6 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_strict(<2 x half> %x, <2 x h
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v2, v3, v2
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10PLUS-SDAG-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10PLUS-GISEL-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <2 x half> %val
}
@@ -348,20 +334,6 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_ignore(<2 x half> %x, <2 x h
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v2, v3, v2
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10PLUS-SDAG-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10PLUS-GISEL-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
ret <2 x half> %val
}
@@ -447,20 +419,6 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_maytrap(<2 x half> %x, <2 x
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v2, v3, v2
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10PLUS-SDAG-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10PLUS-GISEL-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
ret <2 x half> %val
}
@@ -568,27 +526,6 @@ define <3 x half> @v_constained_fsub_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: v_sub_f16_e32 v1.l, v1.l, v3.l
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-SDAG-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v2, v5, v4
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10PLUS-SDAG-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-GISEL-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v2, v4, v5
-; GFX10PLUS-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10PLUS-GISEL-NEXT: s_setpc_b64 s[30:31]
%val = call <3 x half> @llvm.experimental.constrained.fsub.v3f16(<3 x half> %x, <3 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <3 x half> %val
}
@@ -697,36 +634,6 @@ define <4 x half> @v_constained_fsub_v4f16_fpexcept_strict(<4 x half> %x, <4 x h
; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: v_pk_add_f16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-SDAG-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX10PLUS-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v2, v6, v5
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e32 v3, v7, v4
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10PLUS-SDAG-NEXT: v_perm_b32 v1, v3, v1, 0x5040100
-; GFX10PLUS-SDAG-NEXT: s_setpc_b64 s[30:31]
-; GFX10PLUS-GISEL-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX10PLUS-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v2, v4, v6
-; GFX10PLUS-GISEL-NEXT: v_sub_f16_e32 v3, v5, v7
-; GFX10PLUS-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10PLUS-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10PLUS-GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX10PLUS-GISEL-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fsub.v4f16(<4 x half> %x, <4 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <4 x half> %val
}
@@ -865,19 +772,6 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
; GFX12-GISEL-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
-; GFX10PLUS-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
-; GFX10PLUS-SDAG: ; %bb.0:
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e64 v0, s2, s3
-; GFX10PLUS-SDAG-NEXT: s_lshr_b32 s0, s3, 16
-; GFX10PLUS-SDAG-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10PLUS-SDAG-NEXT: v_sub_f16_e64 v1, s1, s0
-; GFX10PLUS-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-SDAG-NEXT: ; return to shader part epilog
-; GFX10PLUS-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
-; GFX10PLUS-GISEL: ; %bb.0:
-; GFX10PLUS-GISEL-NEXT: v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10PLUS-GISEL-NEXT: ; return to shader part epilog
%val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <2 x half> %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/subreg16-allocator.mir b/llvm/test/CodeGen/AMDGPU/subreg16-allocator.mir
new file mode 100644
index 0000000000000..53ea81ac28c04
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/subreg16-allocator.mir
@@ -0,0 +1,85 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -start-before=greedy -stop-after=stack-slot-coloring -stress-regalloc=2 -o - %s | FileCheck -check-prefixes=GFX1100 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -start-before=greedy -stop-after=stack-slot-coloring -stress-regalloc=3 -o - %s | FileCheck -check-prefixes=GFX1100_3 %s
+
+
+---
+name: reuse_reg
+tracksRegLiveness: true
+liveins:
+body: |
+ bb.0:
+ liveins: $vgpr0
+ ; GFX1100-LABEL: name: reuse_reg
+ ; GFX1100: liveins: $vgpr0
+ ; GFX1100-NEXT: {{ $}}
+ ; GFX1100-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX1100-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr0_hi16
+ ; GFX1100-NEXT: %lohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: %lolohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]], 0, %lohi, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: S_ENDPGM 0, implicit %lolohi, implicit [[COPY]]
+ ;
+ ; GFX1100_3-LABEL: name: reuse_reg
+ ; GFX1100_3: liveins: $vgpr0
+ ; GFX1100_3-NEXT: {{ $}}
+ ; GFX1100_3-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX1100_3-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr0_hi16
+ ; GFX1100_3-NEXT: %lohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: %lolohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]], 0, %lohi, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: S_ENDPGM 0, implicit %lolohi, implicit [[COPY]]
+ %0:vgpr_16 = COPY $vgpr0_lo16
+ %1:vgpr_16 = COPY $vgpr0_hi16
+ %lohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0,%0, 0, %1, 0, 0, 0, implicit $mode, implicit $exec
+ %lolohi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0,%0, 0, %lohi, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %lolohi, implicit %0
+...
+
+---
+name: pack_add_two_ways
+tracksRegLiveness: true
+liveins:
+machineFunctionInfo:
+ stackPtrOffsetReg: $sgpr32
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX1100-LABEL: name: pack_add_two_ways
+ ; GFX1100: liveins: $vgpr0, $vgpr1
+ ; GFX1100-NEXT: {{ $}}
+ ; GFX1100-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1100-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1100-NEXT: [[V_PK_ADD_F16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_F16 0, [[COPY]], 0, [[COPY1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: %lo:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY1]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: %temp_lo:vgpr_32 = COPY %lo
+ ; GFX1100-NEXT: %hi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]].hi16, 0, [[COPY1]].hi16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: %temp_hi:vgpr_32 = COPY %hi
+ ; GFX1100-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]]:vgpr_32 = V_PACK_B32_F16_e64 0, %temp_lo, 0, %temp_hi, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100-NEXT: V_CMP_EQ_F32_e32 [[V_PK_ADD_F16_]], [[V_PACK_B32_F16_e64_]], implicit $mode, implicit $exec, implicit-def $vcc_lo
+ ; GFX1100-NEXT: S_ENDPGM 0, implicit [[V_PACK_B32_F16_e64_]]
+ ;
+ ; GFX1100_3-LABEL: name: pack_add_two_ways
+ ; GFX1100_3: liveins: $vgpr0, $vgpr1
+ ; GFX1100_3-NEXT: {{ $}}
+ ; GFX1100_3-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1100_3-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1100_3-NEXT: [[V_PK_ADD_F16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_F16 0, [[COPY]], 0, [[COPY1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: %lo:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY1]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: %temp_lo:vgpr_32 = COPY %lo
+ ; GFX1100_3-NEXT: %hi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY]].hi16, 0, [[COPY1]].hi16, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: %temp_hi:vgpr_32 = COPY %hi
+ ; GFX1100_3-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]]:vgpr_32 = V_PACK_B32_F16_e64 0, %temp_lo, 0, %temp_hi, 0, 0, implicit $mode, implicit $exec
+ ; GFX1100_3-NEXT: V_CMP_EQ_F32_e32 [[V_PK_ADD_F16_]], [[V_PACK_B32_F16_e64_]], implicit $mode, implicit $exec, implicit-def $vcc_lo
+ ; GFX1100_3-NEXT: S_ENDPGM 0, implicit [[V_PACK_B32_F16_e64_]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_PK_ADD_F16 0, %0, 0, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %lo:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0,%0.lo16, 0,%1.lo16,0,0,0, implicit $mode, implicit $exec
+ %temp_lo:vgpr_32 = COPY %lo
+ %hi:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0,%0.hi16, 0,%1.hi16,0,0,0, implicit $mode, implicit $exec
+ %temp_hi:vgpr_32 = COPY %hi
+ %3:vgpr_32 = V_PACK_B32_F16_e64 0, %temp_lo, 0, %temp_hi, 0, 0, implicit $mode, implicit $exec
+ V_CMP_EQ_F32_e32 %2, %3, implicit $mode, implicit $exec, implicit-def $vcc
+ S_ENDPGM 0, implicit %3
+...
+
+
diff --git a/llvm/test/CodeGen/AMDGPU/true16-copy-vgpr16-to-sgpr32.mir b/llvm/test/CodeGen/AMDGPU/true16-copy-vgpr16-to-sgpr32.mir
new file mode 100644
index 0000000000000..149cb2ab63ae7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/true16-copy-vgpr16-to-sgpr32.mir
@@ -0,0 +1,118 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck %s
+
+# Ensure READFIRSTLANE is generated, and that its src is REG_SEQUENCE.
+
+---
+name: test4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test4
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY undef %1:sgpr_128
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY undef %4:sgpr_128
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI [[S_MOV_B32_]], %bb.0, %6, %bb.3
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sreg_32 = PHI [[S_MOV_B32_]], %bb.0, %8, %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY [[PHI]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[PHI]]
+ ; CHECK-NEXT: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY3]], [[COPY]], [[S_MOV_B32_1]], 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[PHI]]
+ ; CHECK-NEXT: [[BUFFER_LOAD_USHORT_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[COPY]], [[S_MOV_B32_1]], 2, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_OFFEN]].lo16
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_OFFEN1]].lo16
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY5]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY6]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[REG_SEQUENCE]].lo16, %subreg.lo16, [[REG_SEQUENCE1]].lo16, %subreg.hi16
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:sgpr_lo16 = COPY [[PHI1]].lo16
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_16 = COPY [[COPY7]]
+ ; CHECK-NEXT: [[V_SUB_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_SUB_NC_U16_t16_e64 0, [[COPY8]], 0, killed [[COPY5]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SUB_NC_U16_t16_e64_]], %subreg.lo16, [[DEF2]], %subreg.hi16
+ ; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE3]], implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 255
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 killed [[V_READFIRSTLANE_B32_]], killed [[S_MOV_B32_2]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK-NEXT: S_CMP_LT_I32 [[S_AND_B32_]], killed [[S_MOV_B32_3]], implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 18
+ ; CHECK-NEXT: S_CMP_LT_I32 [[S_AND_B32_]], killed [[S_MOV_B32_5]], implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.1:
+ successors: %bb.3(0x80000000); %bb.3(100.00%)
+
+ %1:sgpr_128 = COPY undef %150:sgpr_128
+ %131:sreg_32 = S_MOV_B32 0
+ %2:sgpr_128 = COPY undef %151:sgpr_128
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x80000000); %bb.4(100.00%)
+
+ %3:sreg_32 = PHI %131:sreg_32, %bb.1, %183, %bb.5
+ %4:sreg_32 = PHI %131:sreg_32, %bb.1, %182, %bb.5
+
+ bb.4:
+ successors: %bb.6(0x40000000), %bb.5(0x40000000); %bb.5(50.00%), %bb.6(50.00%)
+
+ %154:sreg_32 = S_MOV_B32 0
+ %156:vgpr_32 = COPY %3:sreg_32
+ %162:vgpr_32 = COPY %3:sreg_32
+ %161:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN %162:vgpr_32, %1:sgpr_128, %154:sreg_32, 0, 0, 0, implicit $exec
+ %164:vgpr_32 = COPY %3:sreg_32
+ %163:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN %164:vgpr_32, %1:sgpr_128, %154:sreg_32, 2, 0, 0, implicit $exec
+ %9:vgpr_16 = COPY %161.lo16:vgpr_32
+ %10:vgpr_16 = COPY %163.lo16:vgpr_32
+ %165:sreg_32 = COPY %9:vgpr_16
+ %166:sreg_32 = COPY %10:vgpr_16
+ %12:sreg_32 = S_PACK_LL_B32_B16 %165:sreg_32, %166:sreg_32
+ %167:sgpr_lo16 = COPY %4.lo16:sreg_32
+ %170:vgpr_16 = COPY %167:sgpr_lo16
+ %177:vgpr_16 = V_SUB_NC_U16_t16_e64 0, %170:vgpr_16, 0, killed %9:vgpr_16, 0, 0, implicit $exec
+ %179:sreg_32 = COPY killed %177:vgpr_16
+ %180:sreg_32 = S_MOV_B32 255
+ %13:sreg_32 = S_AND_B32 killed %179:sreg_32, killed %180:sreg_32, implicit-def dead $scc
+ %181:sreg_32 = S_MOV_B32 12
+ S_CMP_LT_I32 %13:sreg_32, killed %181:sreg_32, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.6, implicit $scc
+ S_BRANCH %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.3(0x40000000); %bb.6(50.00%), %bb.3(50.00%)
+
+ %183:sreg_32 = S_MOV_B32 -1
+ %182:sreg_32 = IMPLICIT_DEF
+ %184:sreg_32 = S_MOV_B32 18
+ S_CMP_LT_I32 %13:sreg_32, killed %184:sreg_32, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+ S_BRANCH %bb.6
+
+ bb.6:
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/true16_v2f16_reg_seq.ll b/llvm/test/CodeGen/AMDGPU/true16_v2f16_reg_seq.ll
new file mode 100644
index 0000000000000..bc29a4fe44279
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/true16_v2f16_reg_seq.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs -start-after=safe-stack < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-TRUE16 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs -start-after=safe-stack < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-FAKE16 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -march=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs -start-after=safe-stack < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -march=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs -start-after=safe-stack < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-FAKE16 %s
+
+ ; Function Attrs: nounwind
+ define amdgpu_cs half @_amdgpu_cs_main(<2 x i16> %in0, float %in1, float %in2, float %in3) {
+; SDAG-TRUE16-LABEL: _amdgpu_cs_main:
+; SDAG-TRUE16: ; %bb.0: ; %.entry
+; SDAG-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; SDAG-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.l, v0.l
+; SDAG-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SDAG-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; SDAG-TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v1
+; SDAG-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.l, v1.l
+; SDAG-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v3
+; SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, 0x4300, v0.l
+; SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v1
+; SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v2
+; SDAG-TRUE16-NEXT: s_cbranch_vccnz .LBB0_2
+; SDAG-TRUE16-NEXT: ; %bb.1: ; %.bb1
+; SDAG-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.h, v0.h
+; SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.h, v0.l
+; SDAG-TRUE16-NEXT: s_branch .LBB0_3
+; SDAG-TRUE16-NEXT: .LBB0_2: ; %.bb2
+; SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; SDAG-TRUE16-NEXT: v_sub_f16_e32 v0.l, 2.0, v1.l
+; SDAG-TRUE16-NEXT: s_branch .LBB0_3
+; SDAG-TRUE16-NEXT: .LBB0_3:
+;
+; SDAG-FAKE16-LABEL: _amdgpu_cs_main:
+; SDAG-FAKE16: ; %bb.0: ; %.entry
+; SDAG-FAKE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; SDAG-FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; SDAG-FAKE16-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; SDAG-FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-FAKE16-NEXT: v_cvt_f32_i32_e32 v3, v3
+; SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v4, v1
+; SDAG-FAKE16-NEXT: v_cvt_f16_i16_e32 v1, v0
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-FAKE16-NEXT: v_sub_f16_e32 v5, v1, v4
+; SDAG-FAKE16-NEXT: v_pack_b32_f16 v2, v4, v2
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; SDAG-FAKE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, 0x4300, v5
+; SDAG-FAKE16-NEXT: s_cbranch_vccnz .LBB0_2
+; SDAG-FAKE16-NEXT: ; %bb.1: ; %.bb1
+; SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-FAKE16-NEXT: v_cvt_f16_i16_e32 v0, v0
+; SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
+; SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-FAKE16-NEXT: v_sub_f16_e32 v0, v0, v1
+; SDAG-FAKE16-NEXT: s_branch .LBB0_3
+; SDAG-FAKE16-NEXT: .LBB0_2: ; %.bb2
+; SDAG-FAKE16-NEXT: v_sub_f16_e32 v0, 2.0, v3
+; SDAG-FAKE16-NEXT: s_branch .LBB0_3
+; SDAG-FAKE16-NEXT: .LBB0_3:
+;
+; GISEL-TRUE16-LABEL: _amdgpu_cs_main:
+; GISEL-TRUE16: ; %bb.0: ; %.entry
+; GISEL-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GISEL-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.l, v0.l
+; GISEL-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GISEL-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v1
+; GISEL-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.l, v1.l
+; GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v3
+; GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, 0x4300, v0.l
+; GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v1
+; GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v2
+; GISEL-TRUE16-NEXT: s_cbranch_vccnz .LBB0_2
+; GISEL-TRUE16-NEXT: ; %bb.1: ; %.bb1
+; GISEL-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.h, v0.h
+; GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.h, v0.l
+; GISEL-TRUE16-NEXT: s_branch .LBB0_3
+; GISEL-TRUE16-NEXT: .LBB0_2: ; %.bb2
+; GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GISEL-TRUE16-NEXT: v_sub_f16_e32 v0.l, 2.0, v1.l
+; GISEL-TRUE16-NEXT: s_branch .LBB0_3
+; GISEL-TRUE16-NEXT: .LBB0_3:
+;
+; GISEL-FAKE16-LABEL: _amdgpu_cs_main:
+; GISEL-FAKE16: ; %bb.0: ; %.entry
+; GISEL-FAKE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GISEL-FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GISEL-FAKE16-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v4, v1
+; GISEL-FAKE16-NEXT: v_cvt_f16_i16_e32 v1, v0
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GISEL-FAKE16-NEXT: v_sub_f16_e32 v5, v1, v4
+; GISEL-FAKE16-NEXT: v_pack_b32_f16 v2, v4, v2
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GISEL-FAKE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, 0x4300, v5
+; GISEL-FAKE16-NEXT: s_cbranch_vccnz .LBB0_2
+; GISEL-FAKE16-NEXT: ; %bb.1: ; %.bb1
+; GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-FAKE16-NEXT: v_cvt_f16_i16_e32 v0, v0
+; GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
+; GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-FAKE16-NEXT: v_sub_f16_e32 v0, v0, v1
+; GISEL-FAKE16-NEXT: s_branch .LBB0_3
+; GISEL-FAKE16-NEXT: .LBB0_2: ; %.bb2
+; GISEL-FAKE16-NEXT: v_sub_f16_e32 v0, 2.0, v3
+; GISEL-FAKE16-NEXT: s_branch .LBB0_3
+; GISEL-FAKE16-NEXT: .LBB0_3:
+ .entry:
+ %0 = fptosi float %in1 to i32
+ %1 = fptosi float %in2 to i32
+ %2 = fptosi float %in3 to i32
+ %3 = insertelement <3 x i32> poison, i32 %0, i64 0
+ %4 = insertelement <3 x i32> %3, i32 %1, i64 1
+ %vec3int = insertelement <3 x i32> %4, i32 %2, i64 2
+ %vec3 = sitofp <3 x i32> %vec3int to <3 x half>
+ %vec2 = sitofp <2 x i16> %in0 to <2 x half>
+ %5 = extractelement <2 x half> %vec2, i64 0
+ %6 = extractelement <3 x half> %vec3, i64 0
+ %7 = fsub reassoc nnan nsz arcp contract afn half %5, %6
+ %cnd = fcmp ole half %7, 3.5
+ br i1 %cnd, label %.bb1, label %.bb2
+ .bb1:
+ %8 = extractelement <2 x half> %vec2, i64 1
+ %9 = extractelement <3 x half> %vec3, i64 1
+ %10 = fsub reassoc nnan nsz arcp contract afn half %8, %9
+ %cnd1 = fcmp ole half %10, 3.5
+ br label %.bb3
+ .bb2:
+ %11 = extractelement <3 x half> %vec3, i64 2
+ %12 = fsub reassoc nnan nsz arcp contract afn half 2.0, %11
+ br label %.bb3
+ .bb3:
+ %p1 = phi half [ %10, %.bb1 ], [ %12, %.bb2 ]
+ br label %.exit
+ .exit:
+ ret half %p1
+ }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GISEL: {{.*}}
+; SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.mir b/llvm/test/CodeGen/AMDGPU/v_swap_b16.mir
new file mode 100644
index 0000000000000..65d7cdc8dc874
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.mir
@@ -0,0 +1,409 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -simplify-mir -march=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -run-pass=si-shrink-instructions -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -simplify-mir -march=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -run-pass=si-shrink-instructions -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+
+---
+name: swap_phys_condensed
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_phys_condensed
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+
+---
+name: swap_phys_sparse
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_phys_sparse
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+---
+name: swap_phys_liveout
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_phys_liveout
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+---
+name: swap_phys_liveout_superreg
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_phys_liveout_superreg
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+
+---
+name: swap_phys_overlap_x
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr3_vgpr4
+ ; GCN-LABEL: name: swap_phys_overlap_x
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr3_vgpr4
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr3_vgpr4 = V_ADD_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr3_vgpr4, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ $vgpr3_vgpr4 = V_ADD_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr3_vgpr4, 0, 0, implicit $mode, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+...
+
+---
+name: swap_phys_clobber_y
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16
+ ; GCN-LABEL: name: swap_phys_clobber_y
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ ; GCN-NEXT: $vgpr1_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ ; GCN-NEXT: $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: swap_virt_copy_vgpr_16
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: vgpr_16 }
+ - { id: 1, class: vgpr_16 }
+ - { id: 2, class: vgpr_16 }
+body: |
+ bb.0:
+ ; GCN-LABEL: name: swap_virt_copy_vgpr_16
+ ; GCN: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY [[DEF]]
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = COPY [[DEF1]]
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
+ %0 = IMPLICIT_DEF
+ %1 = IMPLICIT_DEF
+ %2 = COPY %0
+ %0 = COPY %1
+ %1 = COPY %2
+...
+
+---
+name: swap_exact_max_insns_apart
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: swap_exact_max_insns_apart
+ ; GCN: $vgpr0_lo16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr1_hi16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr2_lo16 = COPY $vgpr0_lo16
+ ; GCN-NEXT: $vgpr3_hi16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr0_lo16 = IMPLICIT_DEF
+ $vgpr1_hi16 = IMPLICIT_DEF
+ $vgpr2_lo16 = COPY $vgpr0_lo16
+ $vgpr3_hi16 = IMPLICIT_DEF
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr0_lo16 = COPY $vgpr1_hi16
+ $vgpr1_hi16 = COPY $vgpr2_lo16
+ S_ENDPGM 0
+...
+
+---
+name: swap_too_far
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: swap_too_far
+ ; GCN: $vgpr0_lo16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr1_hi16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr2_lo16 = COPY $vgpr0_lo16
+ ; GCN-NEXT: $vgpr3_hi16 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr4_lo16 = COPY $vgpr3_hi16
+ ; GCN-NEXT: $vgpr3_hi16 = COPY $vgpr4_lo16
+ ; GCN-NEXT: $vgpr0_lo16 = COPY $vgpr1_hi16
+ ; GCN-NEXT: $vgpr1_hi16 = COPY $vgpr2_lo16
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr0_lo16 = IMPLICIT_DEF
+ $vgpr1_hi16 = IMPLICIT_DEF
+ $vgpr2_lo16 = COPY $vgpr0_lo16
+ $vgpr3_hi16 = IMPLICIT_DEF
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr4_lo16 = COPY $vgpr3_hi16
+ $vgpr3_hi16 = COPY $vgpr4_lo16
+ $vgpr0_lo16 = COPY $vgpr1_hi16
+ $vgpr1_hi16 = COPY $vgpr2_lo16
+ S_ENDPGM 0
+...
+
+---
+name: swap_liveness_error_mov
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr5_hi16, $vgpr1_vgpr2
+ ; GCN-LABEL: name: swap_liveness_error_mov
+ ; GCN: liveins: $vgpr5_hi16, $vgpr1_vgpr2
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr6_lo16 = V_MOV_B16_t16_e32 $vgpr1_hi16, implicit $exec
+ ; GCN-NEXT: $vgpr1_hi16, $vgpr5_hi16 = V_SWAP_B16 $vgpr5_hi16, $vgpr1_hi16, implicit $exec
+ ; GCN-NEXT: $vgpr5_vgpr6 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr6_lo16 = V_MOV_B16_t16_e32 $vgpr7_hi16, implicit $exec, implicit $vgpr6_vgpr7
+ ; GCN-NEXT: $vgpr5_hi16 = V_MOV_B16_t16_e32 $vgpr6_lo16, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr6_lo16 = V_MOV_B16_t16_e32 $vgpr1_hi16, implicit $exec, implicit killed $vgpr1_vgpr2
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr5_hi16, implicit $exec
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 $vgpr6_lo16, implicit $exec, implicit-def $vgpr5_vgpr6, implicit $vgpr6_vgpr7
+ $vgpr6_lo16 = V_MOV_B16_t16_e32 $vgpr7_hi16, implicit $exec, implicit $vgpr6_vgpr7
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 $vgpr6_lo16, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: swap_liveness_error_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr5_hi16, $vgpr1_vgpr2
+ ; GCN-LABEL: name: swap_liveness_error_copy
+ ; GCN: liveins: $vgpr5_hi16, $vgpr1_vgpr2
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr6_lo16 = COPY $vgpr1_hi16
+ ; GCN-NEXT: $vgpr1_hi16, $vgpr5_hi16 = V_SWAP_B16 $vgpr5_hi16, $vgpr1_hi16, implicit $exec
+ ; GCN-NEXT: $vgpr5_vgpr6 = IMPLICIT_DEF
+ ; GCN-NEXT: $vgpr6_lo16 = COPY $vgpr7, implicit $vgpr6_vgpr7
+ ; GCN-NEXT: $vgpr5_hi16 = COPY $vgpr6_lo16
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr6_lo16 = COPY $vgpr1_hi16, implicit killed $vgpr1_vgpr2
+ $vgpr1_hi16 = COPY killed $vgpr5_hi16
+ $vgpr5_hi16 = COPY $vgpr6_lo16, implicit-def $vgpr5_vgpr6, implicit $vgpr6_vgpr7
+ $vgpr6_lo16 = COPY $vgpr7, implicit $vgpr6_vgpr7
+ $vgpr5_hi16 = COPY $vgpr6_lo16
+ S_ENDPGM 0
+...
+
+
+---
+name: swap_killed_t_early
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_killed_t_early
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec, implicit killed $vgpr2_lo16
+ ; GCN-NEXT: $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec, implicit killed $vgpr2_lo16
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 undef $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+
+---
+name: swap_killed_t_late
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_killed_t_late
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec, implicit killed $vgpr2_lo16
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec, implicit killed $vgpr2_lo16
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 undef $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+---
+name: swap_killed_x
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: swap_killed_x
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr4_lo16, $vgpr6_lo16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr2_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec, implicit killed $vgpr0_lo16
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr1_hi16
+ $vgpr2_lo16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr4_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 killed $vgpr1_hi16, implicit $exec
+ $vgpr5_hi16 = V_MOV_B16_t16_e32 killed $vgpr6_lo16, implicit $exec, implicit killed $vgpr0_lo16
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr2_lo16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr1_hi16
+...
+
+---
+name: implicit_ops_mov_x_swap_b16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: implicit_ops_mov_x_swap_b16
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec, implicit $vgpr2_lo16, implicit killed $vgpr1_vgpr2
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr1_hi16, implicit $exec, implicit $vgpr2_lo16, implicit killed $vgpr1_vgpr2
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr3_hi16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+
+---
+name: implicit_ops_mov_t_swap_b16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: implicit_ops_mov_t_swap_b16
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec, implicit $vgpr2_lo16, implicit killed $vgpr1_vgpr2, implicit-def $vgpr1_hi16
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec, implicit $vgpr2_lo16, implicit killed $vgpr1_vgpr2, implicit-def $vgpr1_hi16
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr3_hi16, implicit $exec
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
+
+---
+name: implicit_ops_mov_y_swap_b16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr2_lo16, $sgpr30_sgpr31
+ ; GCN-LABEL: name: implicit_ops_mov_y_swap_b16
+ ; GCN: liveins: $vgpr0_lo16, $vgpr1_hi16, $vgpr2_lo16, $sgpr30_sgpr31
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vgpr3_hi16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_lo16, $vgpr1_hi16 = V_SWAP_B16 $vgpr1_hi16, $vgpr0_lo16, implicit $exec
+ ; GCN-NEXT: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+ $vgpr3_hi16 = V_MOV_B16_t16_e32 killed $vgpr0_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr1_hi16, implicit $exec
+ $vgpr1_hi16 = V_MOV_B16_t16_e32 killed $vgpr3_hi16, implicit $exec, implicit $vgpr2_lo16, implicit-def $vgpr0_vgpr1, implicit killed $vgpr3_hi16
+ S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0_lo16, implicit $vgpr1_hi16
+...
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard-fake16.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard-fake16.mir
new file mode 100644
index 0000000000000..0e621c90eff9d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard-fake16.mir
@@ -0,0 +1,20 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -mattr=-real-true16 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN %s
+
+--- |
+ @mem = internal unnamed_addr addrspace(4) constant [4 x <4 x i32>] [<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>]
+
+ define amdgpu_gs void @mask_hazard_cndmask_fake16_dpp4() { ret void }
+---
+name: mask_hazard_cndmask_fake16_dpp4
+body: |
+ bb.0:
+ ; GCN-LABEL: name: mask_hazard_cndmask_fake16_dpp4
+ ; GCN: $vgpr0 = V_CNDMASK_B16_fake16_e64_dpp $vgpr0, 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, 1, 15, 15, 1, implicit $exec
+ ; GCN-NEXT: $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit $scc
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr0 = V_CNDMASK_B16_fake16_e64_dpp $vgpr0, 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, 1, 15, 15, 1, implicit $exec
+ $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit $scc
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index afaf446c583b9..7d2be4fa4b9f5 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -102,11 +102,17 @@ define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i8:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i8:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i8:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -142,15 +148,25 @@ define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i8:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i8:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i8:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %v)
ret i8 %res
@@ -229,13 +245,21 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i8:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -261,17 +285,29 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i8:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %v)
ret i8 %res
@@ -370,15 +406,25 @@ define i8 @test_vector_reduce_mul_v4i8(<4 x i8> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i8:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i8:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i8:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -408,19 +454,33 @@ define i8 @test_vector_reduce_mul_v4i8(<4 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i8:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i8:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i8:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %v)
ret i8 %res
@@ -561,20 +621,35 @@ define i8 @test_vector_reduce_mul_v8i8(<8 x i8> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i8:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i8:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v5.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v6.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i8:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -614,24 +689,43 @@ define i8 @test_vector_reduce_mul_v8i8(<8 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i8:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i8:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v5.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v6.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i8:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %v)
ret i8 %res
@@ -854,30 +948,55 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i8:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v9.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v10.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v11.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v4.l, v12.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.h, v5.l, v13.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.l, v6.l, v14.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.h, v7.l, v15.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v2.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v3.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v1.h, v3.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i8:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -935,34 +1054,63 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i8:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v9.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v10.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v11.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v4.l, v12.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.h, v5.l, v13.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.l, v6.l, v14.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.h, v7.l, v15.l
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v2.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v3.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v1.h, v3.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i8:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %v)
ret i8 %res
@@ -1013,13 +1161,21 @@ define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1045,17 +1201,29 @@ define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %v)
ret i16 %res
@@ -1238,14 +1406,23 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1273,18 +1450,31 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %v)
ret i16 %res
@@ -1405,17 +1595,29 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1449,21 +1651,37 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %v)
ret i16 %res
@@ -1642,22 +1860,39 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) {
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1701,26 +1936,47 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %v)
ret i16 %res
@@ -4228,6 +4484,3 @@ declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>)
declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64>)
declare i64 @llvm.vector.reduce.mul.v16i64(<16 x i64>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-GISEL-FAKE16: {{.*}}
-; GFX12-GISEL-FAKE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index ee346785aee2c..0baf072cf6e52 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -4,6 +4,8 @@
; RUN: llc -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) {
; GFX9-LABEL: shuffle_v4f16_23uu:
@@ -26,6 +28,17 @@ define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_23uu:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
@@ -73,6 +86,18 @@ define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_234u:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 poison>
@@ -100,6 +125,17 @@ define <4 x half> @shuffle_v4f16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_u1u3:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
@@ -139,6 +175,18 @@ define <4 x half> @shuffle_v4f16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_u3u1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 1>
@@ -166,6 +214,17 @@ define <4 x half> @shuffle_v4f16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_u3uu:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
@@ -225,6 +284,34 @@ define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_3u6u:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_3u6u:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 poison, i32 6, i32 poison>
@@ -284,6 +371,34 @@ define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_3uu7:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_3uu7:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 poison, i32 poison, i32 7>
@@ -342,6 +457,34 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_35u5:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_35u5:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
@@ -404,6 +547,35 @@ define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_357u:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_357u:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s0, v3, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 poison>
@@ -434,6 +606,18 @@ define <4 x half> @shuffle_v4f16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_0101:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -461,6 +645,17 @@ define <4 x half> @shuffle_v4f16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_0123:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -497,6 +692,18 @@ define <4 x half> @shuffle_v4f16_0145(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_0145:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -533,6 +740,18 @@ define <4 x half> @shuffle_v4f16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_0167:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
@@ -572,6 +791,18 @@ define <4 x half> @shuffle_v4f16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_2301:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
@@ -602,6 +833,18 @@ define <4 x half> @shuffle_v4f16_2323(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_2323:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
@@ -638,6 +881,18 @@ define <4 x half> @shuffle_v4f16_2345(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_2345:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
@@ -674,6 +929,18 @@ define <4 x half> @shuffle_v4f16_2367(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_2367:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
@@ -712,6 +979,20 @@ define <4 x half> @shuffle_v4f16_4501(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_4501:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
@@ -750,6 +1031,20 @@ define <4 x half> @shuffle_v4f16_4523(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_4523:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
@@ -780,6 +1075,18 @@ define <4 x half> @shuffle_v4f16_4545(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_4545:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5>
@@ -807,6 +1114,17 @@ define <4 x half> @shuffle_v4f16_4567(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_4567:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -845,6 +1163,20 @@ define <4 x half> @shuffle_v4f16_6701(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_6701:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
@@ -883,6 +1215,20 @@ define <4 x half> @shuffle_v4f16_6723(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_6723:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
@@ -922,6 +1268,18 @@ define <4 x half> @shuffle_v4f16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_6745:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5>
@@ -952,6 +1310,18 @@ define <4 x half> @shuffle_v4f16_6767(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4f16_6767:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7>
@@ -1012,6 +1382,35 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_2356:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_2356:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1072,6 +1471,35 @@ define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_5623:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_5623:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -1124,6 +1552,38 @@ define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_3456:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_3456:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -1176,6 +1636,38 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_5634:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_5634:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4>
@@ -1242,6 +1734,39 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_5734:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_5734:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v3, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4>
@@ -1302,6 +1827,35 @@ define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %a
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4i16_2356:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4i16_2356:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, ptr addrspace(1) %arg0
%val1 = load <4 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1338,6 +1892,18 @@ define <4 x i16> @shuffle_v4i16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %a
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4i16_0167:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, ptr addrspace(1) %arg0
%val1 = load <4 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
@@ -1393,6 +1959,34 @@ define <4 x half> @shuffle_v4f16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_0000:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_0000:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer
@@ -1438,6 +2032,35 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_1010:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_1010:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v0, v0, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
@@ -1494,6 +2117,33 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_1100:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_1100:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
@@ -1554,6 +2204,36 @@ define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_6161:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_6161:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1>
@@ -1603,6 +2283,31 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_2333:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_2333:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1652,6 +2357,31 @@ define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_6667:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_6667:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1682,6 +2412,18 @@ define <4 x half> @shuffle_v8f16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8f16_0101:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -1709,6 +2451,17 @@ define <4 x half> @shuffle_v8f16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8f16_0123:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -1745,6 +2498,18 @@ define <4 x half> @shuffle_v8f16_4589(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8f16_4589:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9>
@@ -1783,6 +2548,20 @@ define <4 x half> @shuffle_v8f16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrspace
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8f16_10_11_2_3:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3>
@@ -1843,6 +2622,35 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -1891,6 +2699,30 @@ define <4 x half> @shuffle_v3f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v3f16_0122:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v3f16_0122:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <3 x half>, ptr addrspace(1) %arg0
%val1 = load <3 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
@@ -1930,6 +2762,31 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v2f16_0122:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v2f16_0122:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x half>, ptr addrspace(1) %arg0
%val1 = load <2 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -1992,6 +2849,22 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1)
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v6f16_452367:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX12-NEXT: global_load_b96 v[0:2], v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[3:4], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x half>, ptr addrspace(1) %arg0
%val1 = load <6 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
@@ -2074,6 +2947,28 @@ define amdgpu_kernel void @fma_shuffle_v2f16(ptr addrspace(1) nocapture readonly
; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
; GFX11-NEXT: global_store_b64 v6, v[0:1], s[4:5]
; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: fma_shuffle_v2f16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
+; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0x2
+; GFX12-NEXT: global_load_b64 v[0:1], v6, s[0:1]
+; GFX12-NEXT: global_load_b64 v[2:3], v6, s[2:3]
+; GFX12-NEXT: global_load_b64 v[4:5], v6, s[4:5]
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1]
+; GFX12-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0]
+; GFX12-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
+; GFX12-NEXT: global_store_b64 v6, v[0:1], s[4:5]
+; GFX12-NEXT: s_endpgm
entry:
%tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x()
%tmp12 = zext i32 %tmp1 to i64
@@ -2160,6 +3055,36 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4f16_0456:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4f16_0456:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
@@ -2218,6 +3143,18 @@ define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in,
; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: global_store_b128 v4, v[0:3], s[2:3]
; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: shuffle_scalar_load_v8i32_0123:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b128 s[4:7], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
+; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
+; GFX12-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[2:3]
+; GFX12-NEXT: s_endpgm
%ld8 = load <8 x i32>, ptr addrspace(4) %in, align 16
%id = shufflevector <8 x i32> %ld8, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
store <4 x i32> %id, ptr addrspace(1) %out, align 8
@@ -2271,6 +3208,32 @@ define <2 x half> @low16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: low16bits_v2f16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: low16bits_v2f16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%1 = load <2 x half>, ptr addrspace(1) %x1, align 4
@@ -2326,6 +3289,32 @@ define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: hi16bits_v2f16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: hi16bits_v2f16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%1 = load <2 x half>, ptr addrspace(1) %x1, align 4
@@ -2381,6 +3370,32 @@ define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: low16hi16bits_v2f16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: low16hi16bits_v2f16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%1 = load <2 x half>, ptr addrspace(1) %x1, align 4
@@ -2427,6 +3442,34 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: hi16low16bits_v2bf16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: hi16low16bits_v2bf16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%1 = load <2 x half>, ptr addrspace(1) %x1, align 4
@@ -2482,6 +3525,32 @@ define <2 x i16> @i16_low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: i16_low16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: i16_low16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
@@ -2537,6 +3606,32 @@ define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: i16_low16hi16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: i16_low16hi16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
@@ -2583,6 +3678,34 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: i16_hi16low16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: i16_hi16low16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
@@ -2638,6 +3761,32 @@ define <2 x i16> @i16_hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: i16_hi16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: i16_hi16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
@@ -2670,6 +3819,18 @@ define <2 x i16> @v2i16_hi16bits(ptr addrspace(1) %x0) {
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v2i16_hi16bits:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%load0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%insert1 = insertelement <2 x i16> poison, i16 0, i32 0
@@ -2702,6 +3863,18 @@ define <2 x half> @v2half_hi16bits(ptr addrspace(1) %x0) {
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v2half_hi16bits:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%load0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%insert1 = insertelement <2 x half> poison, half 0.0, i32 0
@@ -2738,6 +3911,19 @@ define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8f16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -2779,6 +3965,21 @@ define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v16f16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -2842,6 +4043,29 @@ define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v32f16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[14:17], v[0:1], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[14:17], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <16 x half>, ptr addrspace(1) %arg0
%val1 = load <16 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <16 x half> %val0, <16 x half> %val1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -2877,6 +4101,19 @@ define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8i16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, ptr addrspace(1) %arg0
%val1 = load <4 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -2918,6 +4155,21 @@ define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v16i16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x i16>, ptr addrspace(1) %arg0
%val1 = load <8 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x i16> %val0, <8 x i16> %val1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -2981,6 +4233,29 @@ define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v32i16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[14:17], v[0:1], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[14:17], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <16 x i16>, ptr addrspace(1) %arg0
%val1 = load <16 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <16 x i16> %val0, <16 x i16> %val1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -3038,6 +4313,34 @@ define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: global_store_b32 v[4:5], v0, off
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4i8_concat:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_store_b32 v[4:5], v0, off
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4i8_concat:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_u16 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_store_b32 v[4:5], v0, off
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x i8>, ptr addrspace(1) %arg0
%val1 = load <2 x i8>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x i8> %val0, <2 x i8> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3073,6 +4376,19 @@ define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1,
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b64 v[4:5], v[0:1], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8i8_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i8>, ptr addrspace(1) %arg0
%val1 = load <4 x i8>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i8> %val0, <4 x i8> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -3108,6 +4424,19 @@ define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v16i8_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x i8>, ptr addrspace(1) %arg0
%val1 = load <8 x i8>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x i8> %val0, <8 x i8> %val1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -3149,6 +4478,21 @@ define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v32i8_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <16 x i8>, ptr addrspace(1) %arg0
%val1 = load <16 x i8>, ptr addrspace(1) %arg1
%shuffle = shufflevector <16 x i8> %val0, <16 x i8> %val1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -3184,6 +4528,19 @@ define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4i32_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x i32>, ptr addrspace(1) %arg0
%val1 = load <2 x i32>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x i32> %val0, <2 x i32> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3225,6 +4582,21 @@ define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8i32_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i32>, ptr addrspace(1) %arg0
%val1 = load <4 x i32>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i32> %val0, <4 x i32> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -3288,6 +4660,29 @@ define void @shuffle_v16i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v16i32_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[14:17], v[0:1], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[14:17], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x i32>, ptr addrspace(1) %arg0
%val1 = load <8 x i32>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x i32> %val0, <8 x i32> %val1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -3316,6 +4711,17 @@ define <4 x bfloat> @shuffle_v4bf16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_23uu:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
@@ -3363,6 +4769,18 @@ define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_234u:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 poison>
@@ -3390,6 +4808,17 @@ define <4 x bfloat> @shuffle_v4bf16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_u1u3:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
@@ -3429,6 +4858,18 @@ define <4 x bfloat> @shuffle_v4bf16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_u3u1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 1>
@@ -3456,6 +4897,17 @@ define <4 x bfloat> @shuffle_v4bf16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_u3uu:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
@@ -3515,6 +4967,34 @@ define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_3u6u:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_3u6u:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 poison, i32 6, i32 poison>
@@ -3574,6 +5054,34 @@ define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_3uu7:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_3uu7:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 poison, i32 poison, i32 7>
@@ -3632,6 +5140,34 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_35u5:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_35u5:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
@@ -3694,6 +5230,35 @@ define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_357u:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_357u:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s0, v3, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 poison>
@@ -3724,6 +5289,18 @@ define <4 x bfloat> @shuffle_v4bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_0101:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -3751,6 +5328,17 @@ define <4 x bfloat> @shuffle_v4bf16_0123(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_0123:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3787,6 +5375,18 @@ define <4 x bfloat> @shuffle_v4bf16_0145(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_0145:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -3823,6 +5423,18 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_0167:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
@@ -3862,6 +5474,18 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_2301:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
@@ -3892,6 +5516,18 @@ define <4 x bfloat> @shuffle_v4bf16_2323(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_2323:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
@@ -3928,6 +5564,18 @@ define <4 x bfloat> @shuffle_v4bf16_2345(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_2345:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
@@ -3964,6 +5612,18 @@ define <4 x bfloat> @shuffle_v4bf16_2367(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_2367:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
@@ -4002,6 +5662,20 @@ define <4 x bfloat> @shuffle_v4bf16_4501(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_4501:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
@@ -4040,6 +5714,20 @@ define <4 x bfloat> @shuffle_v4bf16_4523(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_4523:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
@@ -4070,6 +5758,18 @@ define <4 x bfloat> @shuffle_v4bf16_4545(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_4545:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5>
@@ -4097,6 +5797,17 @@ define <4 x bfloat> @shuffle_v4bf16_4567(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_4567:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -4135,6 +5846,20 @@ define <4 x bfloat> @shuffle_v4bf16_6701(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_6701:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
@@ -4173,6 +5898,20 @@ define <4 x bfloat> @shuffle_v4bf16_6723(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_6723:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
@@ -4212,6 +5951,18 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_6745:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5>
@@ -4242,6 +5993,18 @@ define <4 x bfloat> @shuffle_v4bf16_6767(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v4bf16_6767:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[2:3], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7>
@@ -4302,6 +6065,35 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_2356:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_2356:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -4362,6 +6154,35 @@ define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_5623:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_5623:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -4414,6 +6235,38 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_3456:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_3456:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -4466,6 +6319,38 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_5634:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_5634:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4>
@@ -4532,6 +6417,39 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_5734:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_5734:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v3, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4>
@@ -4587,6 +6505,34 @@ define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_0000:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_0000:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> zeroinitializer
@@ -4632,6 +6578,35 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_1010:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_1010:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v0, v0, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
@@ -4688,6 +6663,33 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_1100:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_1100:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
@@ -4748,6 +6750,36 @@ define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_6161:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[2:3], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_6161:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1>
@@ -4797,6 +6829,31 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_2333:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_2333:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4846,6 +6903,31 @@ define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_6667:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_6667:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4876,6 +6958,18 @@ define <4 x bfloat> @shuffle_v8bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8bf16_0101:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -4903,6 +6997,17 @@ define <4 x bfloat> @shuffle_v8bf16_0123(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8bf16_0123:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -4939,6 +7044,18 @@ define <4 x bfloat> @shuffle_v8bf16_4589(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8bf16_4589:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; GFX12-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9>
@@ -4977,6 +7094,20 @@ define <4 x bfloat> @shuffle_v8bf16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrsp
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8bf16_10_11_2_3:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v[2:3], off offset:4
+; GFX12-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3>
@@ -5037,6 +7168,35 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -5085,6 +7245,30 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v3bf16_0122:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v3bf16_0122:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <3 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <3 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <3 x bfloat> %val0, <3 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
@@ -5124,6 +7308,31 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v2bf16_0122:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v2bf16_0122:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <2 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x bfloat> %val0, <2 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -5186,6 +7395,22 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v6bf16_452367:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX12-NEXT: global_load_b96 v[0:2], v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[3:4], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <6 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <6 x bfloat> %val0, <6 x bfloat> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
@@ -5634,6 +7859,207 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
; GFX11-FAKE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: fma_shuffle_v2bf16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
+; GFX12-TRUE16-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_clause 0x2
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
+; GFX12-TRUE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX12-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_fmac_f32 v0, v8, v4 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_dual_fmac_f32 v7, v3, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_dual_cndmask_b32 v4, v5, v8 :: v_dual_fmac_f32 v1, v3, v10
+; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v11, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX12-TRUE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: fma_shuffle_v2bf16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
+; GFX12-FAKE16-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_clause 0x2
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
+; GFX12-FAKE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
+; GFX12-FAKE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX12-FAKE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
+; GFX12-FAKE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_fmac_f32 v0, v8, v4 :: v_dual_fmac_f32 v7, v8, v9
+; GFX12-FAKE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX12-FAKE16-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX12-FAKE16-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_fmac_f32 v7, v3, v5
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX12-FAKE16-NEXT: v_fmac_f32_e32 v1, v3, v10
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-FAKE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
+; GFX12-FAKE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
+; GFX12-FAKE16-NEXT: s_endpgm
entry:
%tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x()
%tmp12 = zext i32 %tmp1 to i64
@@ -5720,6 +8146,36 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: shuffle_v4bf16_0456:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: shuffle_v4bf16_0456:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b64 v[1:2], v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
@@ -5773,6 +8229,32 @@ define <2 x bfloat> @low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: low16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: low16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
%1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
@@ -5828,6 +8310,32 @@ define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: hi16bits_v2bf16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: hi16bits_v2bf16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
%1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
@@ -5883,6 +8391,32 @@ define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1)
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: low16hi16bits_v2bf16:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: low16hi16bits_v2bf16:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
%1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
@@ -5929,6 +8463,34 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: hi16low16bits:
+; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: hi16low16bits:
+; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
%1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
@@ -5961,6 +8523,18 @@ define <2 x bfloat> @v2bfloat_hi16bits(ptr addrspace(1) %x0) {
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v2bfloat_hi16bits:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%load0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
%insert1 = insertelement <2 x bfloat> poison, bfloat 0.0, i32 0
@@ -5997,6 +8571,19 @@ define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v8bf16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -6038,6 +8625,21 @@ define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %ar
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v16bf16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -6101,6 +8703,29 @@ define void @shuffle_v32bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %ar
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: shuffle_v32bf16_concat:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[6:9], v[2:3], off
+; GFX12-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[14:17], v[0:1], off
+; GFX12-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: global_store_b128 v[4:5], v[14:17], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%val0 = load <16 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <16 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <16 x bfloat> %val0, <16 x bfloat> %val1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-vinterp.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-vinterp.mir
index 41fe81f542c81..22ee92cd01ca6 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-vinterp.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-vinterp.mir
@@ -2,29 +2,57 @@
# RUN: llc -mtriple=amdgpu11.00 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefix=GFX11 %s
---
-name: waitcnt-vinterp
+name: waitcnt-vinterp-f16
machineFunctionInfo:
isEntryFunction: true
body: |
bb.0:
liveins: $vgpr0
- ; GFX11-LABEL: name: waitcnt-vinterp
+ ; GFX11-LABEL: name: waitcnt-vinterp-f16
; GFX11: liveins: $vgpr0
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: $vgpr1 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
; GFX11-NEXT: $vgpr2 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
; GFX11-NEXT: $vgpr3 = LDS_PARAM_LOAD 0, 2, 0, implicit $m0, implicit $exec
; GFX11-NEXT: $vgpr4 = LDS_PARAM_LOAD 0, 3, 0, implicit $m0, implicit $exec
- ; GFX11-NEXT: $vgpr5 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr1, 0, $vgpr0, 0, $vgpr1, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
- ; GFX11-NEXT: $vgpr6 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
- ; GFX11-NEXT: $vgpr7 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 0, 1, implicit $m0, implicit $exec, implicit $mode
- ; GFX11-NEXT: $vgpr8 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr4, 0, $vgpr0, 0, $vgpr4, 0, 0, 0, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr5 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr1_lo16, 0, $vgpr0, 0, $vgpr1_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr6 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr2_lo16, 0, $vgpr0, 0, $vgpr2_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr7 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr3_lo16, 0, $vgpr0, 0, $vgpr3_lo16, 0, 0, 1, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr8 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr4_lo16, 0, $vgpr0, 0, $vgpr4_lo16, 0, 0, 0, implicit $m0, implicit $exec, implicit $mode
$vgpr1 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
$vgpr2 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
$vgpr3 = LDS_PARAM_LOAD 0, 2, 0, implicit $m0, implicit $exec
$vgpr4 = LDS_PARAM_LOAD 0, 3, 0, implicit $m0, implicit $exec
- $vgpr5 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr1, 0, $vgpr0, 0, $vgpr1, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
- $vgpr6 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
- $vgpr7 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
- $vgpr8 = V_INTERP_P10_F16_F32_inreg_fake16 0, $vgpr4, 0, $vgpr0, 0, $vgpr4, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr5 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr1_lo16, 0, $vgpr0, 0, $vgpr1_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr6 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr2_lo16, 0, $vgpr0, 0, $vgpr2_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr7 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr3_lo16, 0, $vgpr0, 0, $vgpr3_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr8 = V_INTERP_P10_F16_F32_inreg_t16 0, $vgpr4_lo16, 0, $vgpr0, 0, $vgpr4_lo16, 0, 0, 2, implicit $m0, implicit $exec, implicit $mode
+...
+
+---
+name: waitcnt-vinterp-f32
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+ ; GFX11-LABEL: name: waitcnt-vinterp-f32
+ ; GFX11: liveins: $vgpr0
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: $vgpr1 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
+ ; GFX11-NEXT: $vgpr2 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
+ ; GFX11-NEXT: $vgpr3 = LDS_PARAM_LOAD 0, 2, 0, implicit $m0, implicit $exec
+ ; GFX11-NEXT: $vgpr4 = LDS_PARAM_LOAD 0, 3, 0, implicit $m0, implicit $exec
+ ; GFX11-NEXT: $vgpr5 = V_INTERP_P10_F32_inreg 0, $vgpr1, 0, $vgpr0, 0, $vgpr1, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr6 = V_INTERP_P10_F32_inreg 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr7 = V_INTERP_P10_F32_inreg 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 1, implicit $m0, implicit $exec, implicit $mode
+ ; GFX11-NEXT: $vgpr8 = V_INTERP_P10_F32_inreg 0, $vgpr4, 0, $vgpr0, 0, $vgpr4, 0, 0, implicit $m0, implicit $exec, implicit $mode
+ $vgpr1 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
+ $vgpr2 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
+ $vgpr3 = LDS_PARAM_LOAD 0, 2, 0, implicit $m0, implicit $exec
+ $vgpr4 = LDS_PARAM_LOAD 0, 3, 0, implicit $m0, implicit $exec
+ $vgpr5 = V_INTERP_P10_F32_inreg 0, $vgpr1, 0, $vgpr0, 0, $vgpr1, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr6 = V_INTERP_P10_F32_inreg 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr7 = V_INTERP_P10_F32_inreg 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 2, implicit $m0, implicit $exec, implicit $mode
+ $vgpr8 = V_INTERP_P10_F32_inreg 0, $vgpr4, 0, $vgpr0, 0, $vgpr4, 0, 2, implicit $m0, implicit $exec, implicit $mode
...
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index a29c5fc84c70c..a3107fb580395 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.70 < %s | FileCheck %s --check-prefixes=GCN,GFX1170
-; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12
-; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-TRUE16
+; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-FAKE16
define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GCN-LABEL: test_wmma_f32_16x16x16_f16_negA:
@@ -424,6 +424,41 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x
; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
; GFX1170-NEXT: global_store_b128 v[10:11], v[12:15], off
; GFX1170-NEXT: s_endpgm
+;
+; GFX12-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: flat_load_b128 v[12:15], v[8:9]
+; GFX12-TRUE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, v14 :: v_dual_mov_b32 v14, v16
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v15, v18
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX12-TRUE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX12-TRUE16-NEXT: s_endpgm
+;
+; GFX12-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: flat_load_b128 v[12:15], v[8:9] offset:16
+; GFX12-FAKE16-NEXT: flat_load_b128 v[16:19], v[8:9]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-FAKE16-NEXT: v_perm_b32 v15, v15, v14, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v14, v13, v12, 0x5040100
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v13, v19, v18, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v12, v17, v16, 0x5040100
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
+; GFX12-FAKE16-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX12-FAKE16-NEXT: s_endpgm
bb:
%C = load <16 x half>, ptr %Caddr
%C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
More information about the llvm-commits
mailing list